{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I worked with: Weiwei Zhang, Kate Shijie Xu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I certify that all solutions are entirely in my own words and that I have not looked at another student’s solutions. I have given credit to all external sources I consulted. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #1 Python Configuration and Data Loading\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "if sys.version_info[0] < 3:\n",
    "    raise Exception(\"Python 3 not detected.\")\n",
    "    \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm\n",
    "from sklearn.metrics import accuracy_score\n",
    "from scipy import io\n",
    "from save_csv import results_to_csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\pagebreak "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #2 Data Partitioning\n",
    "#### I created a reusable function that can be applied to all 3 datasets. The data is randomized then split. The way it is split is the first \"val_amt\" amount of examples get put in the validation set, the rest are put in the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "\n",
    "#first randomized val_amt get puts in validation set, rest in training\n",
    "def shuffle_train_val_split(name, val_amt=0, percent=0):\n",
    "    data = io.loadmat(\"data/%s_data.mat\" % name)\n",
    "    total_num_ex = data[\"training_data\"].shape[0]\n",
    "    shuffle = np.random.permutation(total_num_ex) #shuffle before split\n",
    "    data_xtrain, data_ytrain = data[\"training_data\"][shuffle], \\\n",
    "    data[\"training_labels\"][shuffle] #split\n",
    "    if not val_amt: #spam case\n",
    "        val_amt = int(percent * total_num_ex)\n",
    "    data_xval, data_yval = data_xtrain[:val_amt,:], data_ytrain[:val_amt,:]\n",
    "    data_xtrain, data_ytrain = data_xtrain[val_amt:,:], data_ytrain[val_amt:,:]\n",
    "    return data_xtrain, data_ytrain, data_xval, data_yval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_xtrain, mnist_ytrain, mnist_xval, mnist_yval = \\\n",
    "shuffle_train_val_split(\"mnist\", 10000) \n",
    "spam_xtrain, spam_ytrain, spam_xval, spam_yval = \\\n",
    "shuffle_train_val_split(\"spam\", percent=0.2) \n",
    "cifar10_xtrain, cifar10_ytrain, cifar10_xval, cifar10_yval = \\\n",
    "shuffle_train_val_split(\"cifar10\", 5000) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\pagebreak "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #3 Support Vector Machines: Coding\n",
    "#### I preprocess the MNIST data by normalizing it. I also increase the \"cache_size\" SVC hyperparameter to speed up computing. I also compute and plot the training and validation error rates for MNIST, SPAM, and CIFAR-10 datasets for varying training set sizes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing\n",
    "mnist_xtrain, mnist_xval = (mnist_xtrain-np.mean(mnist_xtrain))/np.std(mnist_xtrain), \\\n",
    "(mnist_xval-np.mean(mnist_xval))/np.std(mnist_xval)\n",
    "#remember to normalize test set as well\n",
    "\n",
    "def train(data_xtrain, data_ytrain, data_xval, data_yval, num_ex, kernel):\n",
    "    data_ytrain = data_ytrain.reshape(-1,)\n",
    "    data_yval = data_yval.reshape(-1,)\n",
    "    model = svm.SVC(kernel=kernel, cache_size=4000)\n",
    "    model.fit(data_xtrain[:num_ex,:],data_ytrain[:num_ex])\n",
    "    return \\\n",
    "    1-accuracy_score(data_ytrain[:num_ex], model.predict(data_xtrain[:num_ex,:])),\\\n",
    "    1-accuracy_score(data_yval[:num_ex],model.predict(data_xval[:num_ex,:]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exs = [100, 200, 500, 1000, 2000, 5000, 10000] #number of examples\n",
    "mnist_train_err = []\n",
    "mnist_val_err = []\n",
    "for num_ex in num_exs:\n",
    "    train_err, val_err = \\\n",
    "    train(mnist_xtrain, mnist_ytrain, mnist_xval, mnist_yval, num_ex, \"linear\")\n",
    "    mnist_train_err.append(train_err)\n",
    "    mnist_val_err.append(val_err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VdW99/HPjyQQZJ6sSqCgpdWAIWAELBQcEKQOWEsrXL0Vq3Kd6lUfbbH1qZYOD61jtVRxrPWqiFot16o4YR1BgiIKFhlEiaGIqEwikPB7/tj7xJ2Tk5yTcE5OSL7v1+u8svfaa++zdjbm5xr2WubuiIiI1KVVtgsgIiJNn4KFiIgkpWAhIiJJKViIiEhSChYiIpKUgoWIiCSlYCEiIkkpWIjUwczWmNlOM+sel77YzNzM+pjZX8LtIZHj3zAzj+y/YGZnR/Z/bmbvm9lWMyszswfD9KVh2lYzqzSzLyP7P2+MexZJRMFCJLn3gUmxHTM7FGgbl+dT4DepXMzMzgD+Exjt7u2BEuA5AHfv7+7tw/SXgAtj++7+uz2/FZGGUbAQSe5e4EeR/TOAv8bluQcoMrNRKVzvcGCuu68CcPd/u/ttaSmpSIYoWIgkNx/oaGaHmFkOcCrwP3F5vgB+B/w2xev9yMwuN7OS8JoiTZqChUhqYrWLY4F/AR8lyDMT6G1m4+q6kLv/D/ATYCzwT+BjM5ua3uKKpJeChUhq7gX+A5hMzSYoANx9B/Dr8GN1Xczd73P30UBn4FxgmpmNTWeBRdJJwUIkBe7+AUFH93eBv9WR9W6gE/C9FK+7y90fApYAA/a0nCKZkpvtAojsRc4Curj7NjNL+N+Ou1eY2dXATbVdxMwmAxuAF4FtBM1R/YEF6S6wSLqoZiGSIndf5e6lKWR9AFhXx/HNwM+BD4HPgT8A57n7y3teSpHMMC1+JCIiyahmISIiSSlYiIhIUgoWIiKSlIKFiIgkldGhs2Z2HPBHIAe4w92nxx0/F7gAqAS2AlPcfVl47AqCoYqVwEXuPreu7+revbv36dMn7fcgItKcLVq06BN375EsX8ZGQ4Xz3bxHMD1CGbAQmBQLBmGeju6+Odw+CTjf3Y8zs0KC4YdDgAOAZ4Fvuntlbd9XUlLipaWpjGoUEZEYM1vk7iXJ8mWyGWoIsNLdV7v7TmAWMD6aIRYoQu2AWOQaD8xy9x3u/j6wMryeiIhkQSaboXoCayP7ZcDQ+ExmdgFwKdAaODpy7vy4c3smOHcKMAWgd+/eaSm0iIjUlMmaRaKJ1Gq0ebn7DHc/CPgZcGU9z73N3UvcvaRHj6RNbiIi0kCZrFmUAb0i+wVAeR35ZwG3NPBcEWkCdu3aRVlZGV9++WW2iyJx8vPzKSgoIC8vr0HnZzJYLAT6mVlfgrn/JxJM8VzFzPq5+4pw93ggtj0HuN/Mrifo4O4HvJ7BsopIGpSVldGhQwf69OmDWZ2ztEsjcnc2btxIWVkZffv2bdA1MhYswtk3LwTmEgydvcvdl5rZNKDU3ecAF5rZaGAX8BnBcpWE+WYDy4AK4IK6RkKJSNPw5ZdfKlA0QWZGt27d2LBhQ4OvkdH3LNz9CeCJuLRfRrb/u45zf0tqS1SKSBOiQNE07elz0RvcO7bAvN9B2aJsl0REpMlSsKjcBf/8PXykF/pE9mYbN26kuLiY4uJi9ttvP3r27Fm1v3PnzpSuceaZZ7J8+fI688yYMYP77rsvHUWul+eff5758+cnz5ghWikvNz/4uWt7dsshInukW7duLF68GICrr76a9u3bc9lll1XL4+64O61aJf7/5Lvvvjvp91xwwQV7XtgGeP755+nevTvDhg3LyverZhELFhUa6ifSHK1cuZIBAwZw7rnnMnjwYNatW8eUKVMoKSmhf//+TJs2rSrviBEjWLx4MRUVFXTu3JmpU6cycOBAjjjiCD7++GMArrzySm688caq/FOnTmXIkCF861vf4tVXXwVg27ZtfP/732fgwIFMmjSJkpKSqkAWdfnll1NYWEhRURE/+9nPAFi/fj2nnHIKJSUlDBkyhPnz57Nq1SruuOMOrrnmGoqLi6u+pzGpZtGqFeS0Vs1CJM1+9b9LWVa+OXnGeig8oCNXndi/3uctW7aMu+++m1tvvRWA6dOn07VrVyoqKjjqqKOYMGEChYWF1c7ZtGkTo0aNYvr06Vx66aXcddddTJ06tca13Z3XX3+dOXPmMG3aNJ566iluvvlm9ttvPx555BHeeustBg8eXOO89evX88QTT7B06VLMjM8//xyAiy66iJ/+9KcMGzaMNWvWcMIJJ/DOO+9w9tln0717dy6++OJ63386KFgA5LZVzUKkGTvooIM4/PDDq/YfeOAB7rzzTioqKigvL2fZsmU1gkXbtm0ZN24cAIcddhgvvfRSwmufcsopVXnWrFkDwMsvv1xVUxg4cCD9+9cMcF27dqVVq1acc845HH/88ZxwwgkAPPvss9X6TT777DO2b8/+/8wqWADk5atmIZJmDakBZEq7du2qtlesWMEf//hHXn/9dTp37szpp5+e8I3z1q1bV23n5ORQUVGR8Npt2rSpkSeV2bzz8vIoLS3lmWeeYdasWdxyyy08/fTTVTWV6Pc3BeqzgKDfomJHtkshIo1g8+bNdOjQgY4dO7Ju3Trmzq1zqZwGGTFiBLNnzwbg7bffZtmyZTXybNmyhc2bN3PCCSdwww038OabbwIwevRoZsyYUZUv1tfRoUMHtmzZkvaypkrBAiCvLVSoZiHSEgwePJjCwkIGDBjAOeecw/Dhw9P+HT/5yU/46KOPKCoq4rrrrmPAgAF06tSpWp5NmzZx/PHHM3DgQI4++miuv/56IBia+8orr1BUVERhYSG33347AOPHj2f27NkMGjQoKx3cGVv8qLHt0eJHM0dB+6/BabPTWyiRFubdd9/lkEMOyXYxsq6iooKKigry8/NZsWIFY8aMYcWKFeTmZrflP9HzSXXxI/VZgGoWIpJWW7du5ZhjjqGiogJ3Z+bMmVkPFHtq7y59uuTmw85t2S6FiDQTnTt3ZtGi5jWFkPosIOzgVs1CRKQ2ChYQDp3VexYiIrVRsAC9lCcikoSCBeilPBGRJBQsIKxZ6KU8kb3dkUceWeMluxtvvJHzzz+/zvPat28PQHl5ORMmTKj12smG599444188cUXVfvf/e53q+Z8aixr1qzh/vvvT/t1FSwgqFmog1tkrzdp0iRmzZpVLW3WrFlMmjQppfMPOOAAHn744QZ/f3yweOKJJ+jcuXODr9cQChaZlNsWdldAZeK5X0Rk7zBhwgQef/xxduwIWgrWrFlDeXk5I0aMqHr3YfDgwRx66KH8/e9/r3H+mjVrGDBgAADbt29n4sSJFBUVceqpp1abzO+8886rmuL8qquuAuCmm26ivLyco446iqOOOgqAPn368MknnwBw/fXXM2DAAAYMGFA1xfmaNWs45JBDOOecc+jfvz9jxoxJOGngQw89xIABAxg4cCAjR44EoLKykssvv5zDDz+coqIiZs6cCcDUqVN56aWXKC4u5oYbbkjL7xX0nkUgN5gIjIrtkNMhu2URaS6enAr/fju919zvUBg3vdbD3bp1Y8iQITz11FOMHz+eWbNmceqpp2Jm5Ofn8+ijj9KxY0c++eQThg0bxkknnVTr2tS33HIL++yzD0uWLGHJkiXVphn/7W9/S9euXamsrOSYY45hyZIlXHTRRVx//fXMmzeP7t27V7vWokWLuPvuu1mwYAHuztChQxk1ahRdunRhxYoVPPDAA9x+++388Ic/5JFHHuH000+vdv60adOYO3cuPXv2rGrWuvPOO+nUqRMLFy5kx44dDB8+nDFjxjB9+nSuvfZaHn/88Yb+lhNSzQKCN7hBw2dFmoFoU1S0Ccrd+fnPf05RURGjR4/mo48+Yv369bVe58UXX6z6o11UVERRUVHVsdmzZzN48GAGDRrE0qVLE04UGPXyyy/zve99j3bt2tG+fXtOOeWUqinP+/btS3FxMVB9mvOo4cOHM3nyZG6//XYqKysBePrpp/nrX/9KcXExQ4cOZePGjaxYsSLF31L9qWYBkdXy1G8hkjZ11AAy6eSTT+bSSy/ljTfeYPv27VU1gvvuu48NGzawaNEi8vLy6NOnT8KpyaMS1Tref/99rr32WhYuXEiXLl2YPHly0uvUNQdfbIpzCKY5T9QMdeutt7JgwQL+8Y9/UFxczOLFi3F3br75ZsaOHVst7wsvvFBnWRpKNQtQzUKkGWnfvj1HHnkkP/7xj6t1bG/atIl9992XvLw85s2bxwcffFDndUaOHMl9990HwDvvvMOSJUuAYIrzdu3a0alTJ9avX8+TTz5ZdU5t04iPHDmSxx57jC+++IJt27bx6KOP8p3vfCfle1q1ahVDhw5l2rRpdO/enbVr1zJ27FhuueUWdu3aBcB7773Htm3bMjaVuWoWoJqFSDMzadIkTjnllGojo0477TROPPFESkpKKC4u5uCDD67zGueddx5nnnkmRUVFFBcXM2TIECBY+W7QoEH079+fAw88sNoU51OmTGHcuHHsv//+zJs3ryp98ODBTJ48ueoaZ599NoMGDUrY5JTI5ZdfzooVK3B3jjnmGAYOHEhRURFr1qxh8ODBuDs9evTgscceo6ioiNzcXAYOHMjkyZO55JJLUv211UlTlAOsfBb+5/vw46eh99D0FkykBdEU5U3bnkxRntFmKDM7zsyWm9lKM6ux0rmZXWpmy8xsiZk9Z2ZfjxyrNLPF4WdOJstJbtgMpSk/REQSylgzlJnlADOAY4EyYKGZzXH36LCBN4ESd//CzM4D/gCcGh7b7u7FmSpfNXmxZigFCxGRRDJZsxgCrHT31e6+E5gFjI9mcPd57h573XE+UJDB8tQu1meh+aFE9lhzadpubvb0uWQyWPQE1kb2y8K02pwFPBnZzzezUjObb2YnZ6KAVXJVsxBJh/z8fDZu3KiA0cS4Oxs3biQ/P7/B18jkaKhEr0Um/BdkZqcDJcCoSHJvdy83swOB583sbXdfFXfeFGAKQO/evRte0qqhs6pZiOyJgoICysrK2LBhQ7aLInHy8/MpKGh4400mg0UZ0CuyXwCUx2cys9HAL4BR7l419au7l4c/V5vZC8AgoFqwcPfbgNsgGA3V4JKqZiGSFnl5efTt2zfbxZAMyGQz1EKgn5n1NbPWwESg2qgmMxsEzAROcvePI+ldzKxNuN0dGA7U/T79nlDNQkSkThmrWbh7hZldCMwFcoC73H2pmU0DSt19DnAN0B54KHyt/kN3Pwk4BJhpZrsJAtr0uFFU6aWahYhInTL6Bre7PwE8EZf2y8j26FrOexU4NJNlq8YsCBiqWYiIJKS5oWJy87VanohILRQsYnK1Wp6ISG0ULGLy8jXrrIhILRQsYnLbqmYhIlILBYsY1SxERGqlYBGT21ZDZ0VEaqFgEZOnobMiIrVRsIhRzUJEpFYKFjGqWYiI1ErBIkYv5YmI1ErBIkYv5YmI1ErBIiavrYbOiojUQsEiJlaz0ApfIiI1KFjE5OWD74bKXdkuiYhIk6NgEZMbLoCkfgsRkRoULGLywgWQ1G8hIlKDgkWMahYiIrVSsIjJbRP8VM1CRKQGBYuYvFjNQsFCRCSegkVMbthnoWAhIlKDgkVMrGah+aFERGpQsIhRzUJEpFYKFjGqWYiI1ErBIkY1CxGRWilYxKhmISJSq4wGCzM7zsyWm9lKM5ua4PilZrbMzJaY2XNm9vXIsTPMbEX4OSOT5QS+es9CNQsRkRoyFizMLAeYAYwDCoFJZlYYl+1NoMTdi4CHgT+E53YFrgKGAkOAq8ysS6bKCnz1BrdqFiIiNWSyZjEEWOnuq919JzALGB/N4O7z3P2LcHc+UBBujwWecfdP3f0z4BnguAyWNaxZmFbLExFJIJPBoiewNrJfFqbV5izgyQaeu+fMtFqeiEgtcjN4bUuQlnBlITM7HSgBRtXnXDObAkwB6N27d8NKGZWXr7mhREQSyGTNogzoFdkvAMrjM5nZaOAXwEnuvqM+57r7be5e4u4lPXr02PMS57ZVzUJEJIFMBouFQD8z62tmrYGJwJxoBjMbBMwkCBQfRw7NBcaYWZewY3tMmJZZqlmIiCSUsWYod68wswsJ/sjnAHe5+1IzmwaUuvsc4BqgPfCQmQF86O4nufunZvZrgoADMM3dP81UWavkttXQWRGRBDLZZ4G7PwE8EZf2y8j26DrOvQu4K3OlSyC3jYbOiogkoDe4o/JUsxARSUTBIio3XzULEZEEFCyi8trqpTwRkQQULKL0Up6ISEIKFlEaOisikpCCRZReyhMRSUjBIko1CxGRhBQsomJ9Fp5wCisRkRZLwSKqamlVjYgSEYlSsIiKLa2qfgsRkWoULKJiNQv1W4iIVKNgEVVVs1CwEBGJUrCIquqzULAQEYlSsIiK1Sw0P5SISDUKFlGqWYiIJJTyehZmlg+cBuwD3O/uGzNWqmyp6uBWzUJEJKo+NYs/EgSXL4HHMlOcLMtTzUJEJJFag4WZ3W9mB0WSugL3AQ8AXTJdsKzIVZ+FiEgidTVDXQn8xszKgV8D1wJzgHzg6swXLQtUsxARSajWYOHuq4H/MLMRwIPAP4Bj3b2ysQrX6FSzEBFJqK5mqC5mdgFQCPwQ2ATMNbMTGqtwjS5Pc0OJiCRSVwf3Y8AOgmane939r8CJwGFmNqcxCtfoYjWLHZuzWw4RkSamrmDRDbifoFO7J4C7b3f3XwH/1Qhla3y5reGAwfDOI7B7d7ZLIyLSZNQVLH4JPAM8CkyNHnD3dZksVFYdcQFsXAnvPZXtkoiINBm1Bgt3/5u7D3f3ke7+bGMWKqsKT4ZOveDVm7NdEhGRJkPTfcTLyYVh58GHr0LZomyXRkSkSchosDCz48xsuZmtNLOpCY6PNLM3zKzCzCbEHas0s8Xhp3E71Af/CNp0gtdUuxARgSTBwsxyzOyShlzYzHKAGcA4guG3k8ysMC7bh8Bkgo70eNvdvTj8nNSQMjRYmw5w2Bmw7O/w2QeN+tUiIk1RncEifAFvfAOvPQRY6e6r3X0nMCv+Wu6+xt2XAE1v6NHQc8Fawfxbsl0SEZGsS6UZ6hUz+5OZfcfMBsc+KZzXE1gb2S8L01KVb2alZjbfzE5OlMHMpoR5Sjds2FCPS6egU08Y8H1446+w/bP0XltEZC+TyhTl3w5/ToukOXB0kvMsQZqnUqhQb3cvN7MDgefN7G13X1XtYu63AbcBlJSU1OfaqTniQljyICz6C4xoUGuciEizkDRYuPtRDbx2GdArsl8AlKd6sruXhz9Xm9kLwCBgVZ0npdv+RdB3FCyYCcMuCF7aExFpgZI2Q5lZJzO7PtbcY2bXmVmnFK69EOhnZn3NrDUwkWDW2qTCeanahNvdgeHAslTOTbtvXwRb1gVvdYuItFCp9FncBWwhmEzwh8Bm4O5kJ7l7BXAhMBd4F5jt7kvNbJqZnQRgZoebWRnwA2CmmS0NTz8EKDWzt4B5wHR3z06w+MYx0OMQeO1P4Olv6RIR2RuYJ/kDaGaL3b04WVq2lZSUeGlpaWYu/sa9MOdC+M/H4KCGtsoBn6yEl66DdW/BqMuDt8UtUdeOiEjjMLNF7l6SLF8qNYvt4ZoWsQsPB1rWgg9FP4R2+zZ8CpCP34WHz4IZh8PSv0HlTnhoMjx4OmxZn9aiiohkQiqjoc4F/hrpp/gMOCNzRWqCctvA0Cnw/G9g/TL4Wvy7hbVYtwRevAbenQN57eDbPwlGWLXtGrwdPu//wYwhcNx0GDhRtQwRabKSvcHdCviWuw8EioAidx8UvkjXspScBXn7BH0XyZQtgvsnwszvwOoXYOTlcMk7cOw0aL9vMP/UiEvgvFegx8Hw2Llw3wT4fG3SS4uIZEOyN7h3E3RS4+6b3b3lrgq0T1coPg2WzIYt/06c54PX4N7vwR1Hw4evwVG/gIvfhqOvDM6P170fnPkkjPtDcO6fh8HCO7WWhog0Oan0WTxjZpeZWS8z6xr7ZLxkTdER58PuiuC9ixh3WP1P+MsJcPdxQdPT6KuDmsSon0LbznVfs1UrGPpfcP6r0PMw+MelcM+JsLFxXykREalLKqOh3k+Q7O5+YGaK1DAZHQ0V9eDp8P5LcMlS+HA+vPgHWLsA2u8Hwy+CwyZD63YNu7Y7vHkvzP0FVO4KaiTDzoNWOWm9BRGRmFRHQ9UZLMI+iyPc/ZV0Fi4TGi1YrH0d7jwWOuwfvKzXsQBGXAyD/hPy8tPzHZvL4fFL4b0noWcJjJ8B+x6cnmuLiESkZehs2GdxbdpK1Rz0GgIHHgW5+XDiTXDRmzDknPQFCoCOB8CkB+D7d8Knq4OO8n9eE9Q2RESyIJVmqF8BS4C/ebLMWdRoNQsImosaa5jr1g3w5E+D9zP2OzSoZew/sHG+W0SavXS+lHcp8BCw08w2m9kWM2u5o6Kgcd+HaN8DfnA3nHofbP0YbjsKnpsGu75svDKISIuXNFi4ewd3b+Xuee7eMdzv2BiFk4hDToALFgQv7710XdA0tfb1bJdKRFqIVGadNTM73cz+b7jfy8yGZL5oUkPbLnDyn+H0R2DXdrhzDDx1Bezclu2SiUgzl0oz1J+BI4D/CPe3EqytLdnyjdFw/mtw+Fkw/89wy7fh/RezXSoRacZSCRZD3f0C4EsAd/8M0CpA2damAxx/HUz+R7BW+D0nwv9eDF+27O4kEcmMVILFLjPLIVwS1cx6AJqPoqnoMwLOfSWYpPCNe4IpQ957OtulEpFmJpVgcRPwKLCvmf0WeBn4XUZLJfXTeh8Y8xs461lo0xHu/wH87b/gi0+zXTIRaSaSvmcBYGYHA8cABjzn7u9mumD11ajvWTRlFTvgxWvh5euDqdCPvxYKx2e7VCLSRKVluo+9iYJFnH+/DX+/IFiV75CTgv6N9vtmu1Qi0sSk86U82Rvtdyic/TwccxW8NzdYZOmtWVpHXEQaRDWLlmDDe8Ea4msXwL6F0O0b0Lk3dCqATr2gc6/gZ9suWq1PpIVJtWaRyrKqsrfr8c1gkaXSu2D5E7DhX7DiGaiIW0q9dfsgaHQq+CqAdO79VUBpv1+w/oaItDgKFi1Fq5xgdtwh5wT77vDFRvj8Q9i0NljSdVNZuP0hfFQK2z+Lu0ZeMCNuNIBEf3YqCNYrF5FmR8GipTKDdt2DT8/BifPs2BoJJGurb69+IVjPg7hmzPZfiwsgcUElv1Om70xEMkDBQmrXpj3se0jwSaRiJ2wp/yqAfL4WNn0Y/Fy3BP71BFTuiLtmp+o1kfjmrnY91NQl0gQpWEjD5baGLn2CTyK7d8O2DV81bW0Km7piweWDV2HHpurn5LSBTj0jNZLe1YNLx57B94pIo8posDCz44A/AjnAHe4+Pe74SOBGoAiY6O4PR46dAVwZ7v7G3e/JZFklA1q1gg5fCz4FtQy2+HJTzf6SWC1lxTOwdX3cCRYsaVutv6SgelBp0z7jtybS0mQsWITzSc0AjgXKgIVmNsfdl0WyfQhMBi6LO7crcBVQQtAovig8N67HVfZ6+Z1gv06w34DExyt2RAJJXL/JR6Ww7O+wO2652bZdIk1bccODO/eGfbppiLBIPWWyZjEEWOnuqwHMbBYwHqgKFu6+JjwWPzHhWOAZd/80PP4McBzwQAbLK01RbhvodlDwSWR3ZVD7SNQJv3FV0BG/c2vcNdvG9ZdEm7sKoMMBkKMWWpGoTP4X0RNYG9kvA4buwbk94zOZ2RRgCkDv3r0bVkrZu7XKCYbzdjyAhP+83IMhwDVqJ2Fz17ol8MUn1c+x8Jrxw4JjQaVTQTB5o0gLkslgkaien+rr4imd6+63AbdB8AZ36kWTFsMM9ukafPYvSpxn1/aw4z3BOycfvAabHwavrH7OPt1rHx6st+GlGcpksCgDekX2C4Dyepx7ZNy5L6SlVCLx8tpC937BJ5HKiuCdkqrRXJGgUufb8AVBZ3xufjCCK6d1MNorN/ozUVqbIL2+aTmtNexYMiaTwWIh0M/M+gIfARP5amnWZOYCvzOzLuH+GOCK9BdRJAU5uUGNoXOvxMdjb8Mn6oTfuj44Vrkz6Kyv3Blu7wzeQanYQeoV7hS0ygv6eXLy4oJQfdJaxwWmeqZVO9YGWuWqltUMZCxYuHuFmV1I8Ic/B7jL3Zea2TSg1N3nmNnhBAsrdQFONLNfuXt/d//UzH5NEHAApsU6u0WanOjb8AcMqt+57rC7onowqRZUdqR+rD5pO7ZG0nZA5a7q+eJHmO3ZL6hmAKn6mVd3oGlIWkr526gWVk+adVZEatq9OwgY1QJNgqBSuSOsJe2sJW1HpBZVV1r8udHviqR5Gld0bpVbM4BUNenlxQWaeqRVNS8mSkvSDJmT1+i1MM06KyIN16oVtGrT9CaG3F2ZWlBJGJjCvPVN2/5Z5Lt2xn1HmJZO9eqzCoNV92/CkVPTW444ChYisvdolRMOW25CQ5fdawac2oJKtYC0My4w1SNt13bY/vlXtbJ0B6wEFCxERPaEWfB/+rmtoYlVxNJJPTwiIpKUgoWIiCSlYCEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSSlYiIhIUgoWIiKSlIKFiIgkpWAhIiJJKViIiEhSChYiIpKUgoWIiCSlYCEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSSlYiIhIUgoWIiKSlIKFiIgkldFgYWbHmdlyM1tpZlMTHG9jZg+GxxeYWZ8wvY+ZbTezxeHn1kyWU0RE6pabqQubWQ4wAzgWKAMWmtkcd18WyXYW8Jm7f8PMJgK/B04Nj61y9+JMlU9ERFKXyZrFEGClu692953ALGB8XJ7xwD3h9sPAMWZmGSyTiIg0QCaDRU9gbWS/LExLmMfdK4BNQLfwWF8ze9PM/mlm30n0BWY2xcxKzax0w4YN6S29iIhUyWSwSFRD8BTzrAN6u/sg4FLgfjPrWCOj+23uXuLuJT169NjjAouISGKZDBZlQK/IfgFQXlseM8sFOgGfuvsOd98I4O6LgFXANzNYVhERqUMmg8VCoJ+Z9TWz1sBEYE5cnjlNP4EMAAAKK0lEQVTAGeH2BOB5d3cz6xF2kGNmBwL9gNUZLKuIiNQhY6Oh3L3CzC4E5gI5wF3uvtTMpgGl7j4HuBO418xWAp8SBBSAkcA0M6sAKoFz3f3TTJVVRETqZu7x3Qh7p5KSEi8tLc12MURE9ipmtsjdS5Ll0xvcIiKSlIKFiIgkpWAhIiJJKViIiEhSChYiIpKUgoWIiCSlYCEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSSlYiIhIUgoWIiKSlIKFiIgkpWAhIiJJKViIiEhSChYiIpKUgoWIiCSlYCEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSSlYiIhIUgoWIiKSVEaDhZkdZ2bLzWylmU1NcLyNmT0YHl9gZn0ix64I05eb2dhMllNEROqWsWBhZjnADGAcUAhMMrPCuGxnAZ+5+zeAG4Dfh+cWAhOB/sBxwJ/D64mISBbkZvDaQ4CV7r4awMxmAeOBZZE844Grw+2HgT+ZmYXps9x9B/C+ma0Mr/daJgr6q/9dyrLyzZm4tIhIxhUe0JGrTuyf0e/IZDNUT2BtZL8sTEuYx90rgE1AtxTPxcymmFmpmZVu2LAhjUUXEZGoTNYsLEGap5gnlXNx99uA2wBKSkpqHE9VpiOyiMjeLpM1izKgV2S/ACivLY+Z5QKdgE9TPFdERBpJJoPFQqCfmfU1s9YEHdZz4vLMAc4ItycAz7u7h+kTw9FSfYF+wOsZLKuIiNQhY81Q7l5hZhcCc4Ec4C53X2pm04BSd58D3AncG3Zgf0oQUAjzzSboDK8ALnD3ykyVVURE6mbB/8jv/UpKSry0tDTbxRAR2auY2SJ3L0mWT29wi4hIUgoWIiKSlIKFiIgkpWAhIiJJNZsObjPbAHxQz9O6A59koDhNWUu8Z2iZ990S7xla5n3vyT1/3d17JMvUbIJFQ5hZaSqjAJqTlnjP0DLvuyXeM7TM+26Me1YzlIiIJKVgISIiSbX0YHFbtguQBS3xnqFl3ndLvGdomfed8Xtu0X0WIiKSmpZesxARkRQoWIiISFItMliY2XFmttzMVprZ1GyXZ0+YWS8zm2dm75rZUjP77zC9q5k9Y2Yrwp9dwnQzs5vCe19iZoMj1zojzL/CzM6o7TubEjPLMbM3zezxcL+vmS0I7+HBcHp8wunuHwzve4GZ9Ylc44owfbmZjc3OnaTGzDqb2cNm9q/wmR/REp61mV0S/vt+x8weMLP85vaszewuM/vYzN6JpKXt2ZrZYWb2dnjOTWaWaJG52rl7i/oQTJe+CjgQaA28BRRmu1x7cD/7A4PD7Q7Ae0Ah8Adgapg+Ffh9uP1d4EmC1QiHAQvC9K7A6vBnl3C7S7bvL4X7vxS4H3g83J8NTAy3bwXOC7fPB24NtycCD4bbheG/gTZA3/DfRk6276uO+70HODvcbg10bu7PmmBJ5feBtpFnPLm5PWtgJDAYeCeSlrZnS7Am0BHhOU8C4+pVvmz/grLwQI4A5kb2rwCuyHa50nh/fweOBZYD+4dp+wPLw+2ZwKRI/uXh8UnAzEh6tXxN8UOwguJzwNHA4+F/BJ8AufHPmmBdlSPC7dwwn8U//2i+pvYBOoZ/NC0uvVk/6zBYrA3/AOaGz3psc3zWQJ+4YJGWZxse+1ckvVq+VD4tsRkq9g8vpixM2+uF1e1BwALga+6+DiD8uW+Yrbb73xt/LzcCPwV2h/vdgM/dvSLcj95D1f2FxzeF+fem+z4Q2ADcHTa93WFm7Wjmz9rdPwKuBT4E1hE8u0U072cdk65n2zPcjk9PWUsMFona6fb68cNm1h54BLjY3TfXlTVBmteR3iSZ2QnAx+6+KJqcIKsnObY33XcuQTPFLe4+CNhG0DRRm+Zwz4Tt9OMJmo4OANoB4xJkbU7POpn63uMe33tLDBZlQK/IfgFQnqWypIWZ5REEivvc/W9h8noz2z88vj/wcZhe2/3vbb+X4cBJZrYGmEXQFHUj0NnMYssFR++h6v7C450IlvLdm+67DChz9wXh/sMEwaO5P+vRwPvuvsHddwF/A75N837WMel6tmXhdnx6ylpisFgI9AtHUrQm6ACbk+UyNVg4ouFO4F13vz5yaA4QGwlxBkFfRiz9R+FoimHAprB6OxcYY2Zdwv+TGxOmNUnufoW7F7h7H4Jn+Ly7nwbMAyaE2eLvO/b7mBDm9zB9YjiCpi/Qj6AjsMlx938Da83sW2HSMQTr1DfrZ03Q/DTMzPYJ/73H7rvZPuuItDzb8NgWMxsW/g5/FLlWarLdoZOlTqTvEowaWgX8Itvl2cN7GUFQnVwCLA4/3yVoo30OWBH+7BrmN2BGeO9vAyWRa/0YWBl+zsz2vdXjd3AkX42GOpDgD8BK4CGgTZieH+6vDI8fGDn/F+HvYzn1HCGShXstBkrD5/0YwYiXZv+sgV8B/wLeAe4lGNHUrJ418ABBn8wugprAWel8tkBJ+PtbBfyJuIESyT6a7kNERJJqic1QIiJSTwoWIiKSlIKFiIgkpWAhIiJJKViIiEhSucmziDRPZvb/CMaldwYOdvfpWS5SrczsamCru1+b7bJIy6SahbRkQwnm0RoFvJTlsog0aQoW0uKY2TVmtgQ4HHgNOBu4xcx+mSBvDzN7xMwWhp/hYfpNsfxmNtbMXjSzVmZ2YriGwptm9qyZfS3Mc7WZ3WNmT5vZGjM7xcz+EK4v8FQ4ZQvhsd+b2evh5xsJynRQeM4iM3vJzA4O039gwXoPb5nZi5n6/UkLle23FvXRJxsfYAhwM5AHvFJHvvuBEeF2b4JpVQD2AZYCRxG8DXxQmN6Fr9a2Pxu4Lty+Gng5/L6BwBeEbxADjwInh9trCGcVIJiS4fHI+ZeF288B/cLtoQTTWUDwJm/PcLtztn/H+jSvj/ospKUaRDA1ysEE8wzVZjRQGFlUrKOZdXD3LWZ2DvAicIm7rwqPFwAPhpO+tSZYfyLmSXffZWZvEyzC9VSY/jbBOgYxD0R+3hAtTDi78LeBhyJlahP+fAX4i5nNJphsTyRtFCykRTGzYuAvBH/UPyGoIZiZLSZYCGd73CmtakkHOBTYSDBtdszNwPXuPsfMjiSoEcTsAHD33Wa2y91jc+3spvp/i17Ldqw8n7t7cXxh3P1cMxsKHA8sNrNid9+YoNwi9aY+C2lR3H1x+Ic2tvzs88BYdy+uJSA8DVwY2wmDDWb2deD/ENRQxoV/pCGYDvujcPsMGubUyM/X4sq/GXjfzH4QlsPMbGC4fZC7L3D3XxIEwuhU1SJ7RMFCWhwz6wF85u67CYbM1tUMdRFQYmZLzGwZcG5kWvjL3L2cYHbQO8wsn6Am8ZCZvUTwB7sh2pjZAuC/gUsSHD8NOMvM3iLoNxkfpl8Tdpi/Q9A89lYDv1+kBs06K9KEhIs5lbh7QwONSEaoZiEiIkmpZiEiIkmpZiEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSf1/aTiUHYKgaksAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(num_exs,mnist_train_err,label=\"Training set\")\n",
    "plt.plot(num_exs,mnist_val_err,label=\"Validation set\")\n",
    "plt.xlabel(\"# examples\")\n",
    "plt.ylabel(\"error %\")\n",
    "plt.title(\"MNIST\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exs = [100, 200, 500, 1000, 2000, spam_xtrain.shape[0]] #number of examples\n",
    "spam_train_err = []\n",
    "spam_val_err = []\n",
    "for num_ex in num_exs:\n",
    "    train_err, val_err = \\\n",
    "    train(spam_xtrain, spam_ytrain, spam_xval, spam_yval, num_ex, \"linear\")\n",
    "    spam_train_err.append(train_err)\n",
    "    spam_val_err.append(val_err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VVW6+PHvm0YICaTQIQHEQokpGAIIgm1ULOA4OII6in0s453r1dEp18L8vOPYR8drG0AdEUZFHVTs5SogvUkRARNCCDWUQAqkvL8/9k44CUlOgJzsk+T9PM95zjn7rL3Pe3aS82attddaoqoYY4wx9QnxOgBjjDHBz5KFMcYYvyxZGGOM8cuShTHGGL8sWRhjjPHLkoUxxhi/LFkYY4zxy5KFMQ0gIiNEZJ6I7BOR3SIyV0QGex2XMU0lzOsAjAl2ItIe+AC4FXgTiADOAA56GZcxTclqFsb4dzKAqk5X1XJVLVbVT1V1pYhMdGsZz7q1jh9E5JzKHUXkOhFZKyL7ReQnEbnF57UzRSRXRH4nIjtEZKuIXCoiF4rIj24N5g9efGBjarJkYYx/PwLlIvKqiIwWkbgarw8BfgI6Ag8A74hIvPvaDuBioD1wHfCUiAzy2bcrEAn0AO4HXgauBk7Dqb3cLyInBOZjGdNwliyM8UNVC4ARgOJ8me8UkVki0sUtsgN4WlVLVfVfwDrgInffD1V1ozr+D/gUJwlUKgUeVtVSYAZOwvmbqu5X1dXAaiClCT6mMfWyZGFMA6jqWlWdqKo9gWSgO/C0+/IWrT4j5yb3ddyayHy3SWkvcCFOQqiUr6rl7uNi9367z+vFQHQjfxxjjpolC2OOkqr+ALyCkzQAeoiI+BRJAvJEpA0wE3gc6KKqscBswLesMc2CJQtj/BCRfiLyXyLS032eCEwA5rtFOgN3iki4iFwO9MdJChFAG2AnUCYio4HzmvwDGNMILFkY499+nE7sBSJSiJMkVgH/5b6+ADgJ2AU8DIxT1XxV3Q/ciXO57R7gSmBWE8duTKMQW/zImGMnIhOBG1V1hNexGBNIVrMwxhjjlyULY4wxflkzlDHGGL+sZmGMMcavFjORYMeOHbV3795eh2GMMc3KkiVLdqlqJ3/lWkyy6N27N4sXL/Y6DGOMaVZEZFNDylkzlDHGGL8sWRhjjPHLkoUxxhi/WkyfhTHGe6WlpeTm5lJSUuJ1KKaGyMhIevbsSXh4+DHtb8nCGNNocnNziYmJoXfv3lSfiNd4SVXJz88nNzeXPn36HNMxAtoMJSIXiMg6EdkgIvfV8vpdIrJGRFaKyBci0svdniYi34nIave1KwIZpzGmcZSUlJCQkGCJIsiICAkJCcdV4wtYshCRUOA5YDQwAJggIgNqFFsGZKhqCvA28Ki7vQi4RlUHAhcAT4tIbKBiNcY0HksUwel4fy6BrFlkAhtU9SdVPYSzZORY3wKq+pWqFrlP5wM93e0/qup693EezrKVfgeNBMTezbD6XbBpUYwxrVggk0UPYLPP81x3W11uAD6quVFEMnEWkdlYy2s3i8hiEVm8c+fO4wy3Dp/8Ht6aCJ/9tyUMY4JYfn4+aWlppKWl0bVrV3r06FH1/NChQw06xnXXXce6devqLfPcc88xbdq0xgj5qHz55ZfMnz/ff8EACWQHd211nlq/bUXkaiADGFVjezfgn8C1qlpxxMFUXwJeAsjIyGj8b/KSAvjxU4jpBvOehbJDMPqvYNVsY4JOQkICy5cvB+DBBx8kOjqau+++u1oZVUVVCQmp/f/kqVOn+n2f22+//fiDPQZffvklHTt2ZOjQoZ68fyBrFrlAos/znkBezUIici7wR2CMqh702d4e+BD4k6p6k07XzYbyg3D5qzDsDlj4InzwW6g4Im8ZY4LUhg0bSE5O5te//jWDBg1i69at3HzzzWRkZDBw4EAmTZpUVXbEiBEsX76csrIyYmNjue+++0hNTWXYsGHs2LEDgD/96U88/fTTVeXvu+8+MjMzOeWUU5g3bx4AhYWF/OIXvyA1NZUJEyaQkZFRlch83XPPPQwYMICUlBTuvfdeALZv385ll11GRkYGmZmZzJ8/n40bN/KPf/yDxx57jLS0tKr3aUqBrFksAk4SkT7AFmA8zrKSVUQkHXgRuEBVd/hsjwDeBV5T1bcCGGP9Vs2EDkmQmOncwtrAt084NYyxf4eQUM9CMybYPfT+atbkFTTqMQd0b88Dlww86v3WrFnD1KlTeeGFFwB45JFHiI+Pp6ysjLPOOotx48YxYED162/27dvHqFGjeOSRR7jrrruYMmUK9913xEWdqCoLFy5k1qxZTJo0iY8//phnn32Wrl27MnPmTFasWMGgQYOO2G/79u3Mnj2b1atXIyLs3bsXgDvvvJPf/e53DB06lOzsbC6++GJWrVrFjTfeSMeOHfntb3971J+/MQQsWahqmYjcAXwChAJTVHW1iEwCFqvqLOAxIBp4y+2pz1HVMcAvgZFAgrtsJcBEVT0yNQdK0W7Y+CUMu/1ws9M590NYJHz1MJQfgp+/CKE2VMWYYNe3b18GDx5c9Xz69OlMnjyZsrIy8vLyWLNmzRHJom3btowePRqA0047jW+//bbWY1922WVVZbKzswGYM2dOVU0hNTWVgQOPTHDx8fGEhIRw0003cdFFF3HxxRcD8Pnnn1frN9mzZw/FxcXH+MkbT0C/6VR1NjC7xrb7fR6fW8d+rwOvBzI2v9bOgooySP5F9e2jfgeh4fD5g07C+MVkCIvwJERjgtmx1AACpV27dlWP169fz9/+9jcWLlxIbGwsV199da3jDyIiDv9dh4aGUlZWVuux27Rpc0SZhiwqFx4ezuLFi/nss8+YMWMGzz//PJ9++mlVTcX3/YOBzQ1Vl1UzIeFE6Jpy5Gsj/hPO/4uTUN68BsoOHlnGGBOUCgoKiImJoX379mzdupVPPvmk0d9jxIgRvPnmmwB8//33rFmz5ogy+/fvp6CggIsvvpinnnqKZcuWAXDuuefy3HPPVZWr7OuIiYlh//79jR5rQ1myqM3+7ZA9BwZeVveVT8Nug4uegB8/gukToNT7aqIxxr9BgwYxYMAAkpOTuemmmxg+fHijv8dvfvMbtmzZQkpKCk888QTJycl06NChWpl9+/Zx0UUXkZqaytlnn82TTz4JOJfmzp07l5SUFAYMGMDLL78MwNixY3nzzTdJT0/3pIO7xazBnZGRoY22+NGCl+Cje+C2BdC5X/1ll/4TZv0Geo+AK/8FEe3qL29MC7Z27Vr69+/vdRieKysro6ysjMjISNavX895553H+vXrCQvzto+ztp+PiCxR1Qx/+1rvbG1WzYTOA/0nCoBBv4LQCHjv1/D6L+DKNyGyfeBjNMYErQMHDnDOOedQVlaGqvLiiy96niiOV/OOPhD2bobN8+Hs/274PqlXOJ3eM2+Ef/4crp4JbW0qK2Naq9jYWJYsWeJ1GI3K+ixqWv2uc5982dHtl3wZ/PI12LoCXhvjXHprjDEthCWLmlbNhO6DIP6Eo9+3/8Uw/g3Y8QO8egkcCNB8VcYY08QsWfjK3whblx85tuJonHye09GdvxFeuQj2b2u8+IwxxiOWLHytese5H3jp8R2n71lw9duwLxemXgj7thx/bMYY4yFLFr5WvwNJw6BDz+M/Vu8R8Kt3oXAnTB0NezYd/zGNMfU688wzjxhk9/TTT3PbbbfVu190dDQAeXl5jBs3rs5j+7s8/+mnn6aoqKjq+YUXXlg151NTyc7O5o033mj041qyqLR9DexYc3xNUDUlDYFr3oOSvU6TVP4RS3IYYxrRhAkTmDFjRrVtM2bMYMKECQ3av3v37rz99tvH/P41k8Xs2bOJjW3aKyMtWQTa6ndAQmDAWP9lj0aP0+DaD+BQoZMwdv7YuMc3xlQZN24cH3zwAQcPOlPwZGdnk5eXx4gRI6rGPgwaNIhTTz2Vf//730fsn52dTXJyMgDFxcWMHz+elJQUrrjiimqT+d16661VU5w/8MADADzzzDPk5eVx1llncdZZZwHQu3dvdu3aBcCTTz5JcnIyycnJVVOcZ2dn079/f2666SYGDhzIeeedV+ukgW+99RbJycmkpqYycuRIAMrLy7nnnnsYPHgwKSkpvPjiiwDcd999fPvtt6SlpfHUU081ynkFG2fhUHWuguozEqI7N/7xu6XAxA+dS2pfuRCumQVdai5HbkwL89F9sO37xj1m11Nh9CN1vpyQkEBmZiYff/wxY8eOZcaMGVxxxRWICJGRkbz77ru0b9+eXbt2MXToUMaMGVPn2tTPP/88UVFRrFy5kpUrV1abZvzhhx8mPj6e8vJyzjnnHFauXMmdd97Jk08+yVdffUXHjh2rHWvJkiVMnTqVBQsWoKoMGTKEUaNGERcXx/r165k+fTovv/wyv/zlL5k5cyZXX311tf0nTZrEJ598Qo8ePaqatSZPnkyHDh1YtGgRBw8eZPjw4Zx33nk88sgjPP7443zwwQfHepZrZTULcK6A2v1T4zZB1dRlAEycDRLq1DC2rgjcexnTivk2Rfk2Qakqf/jDH0hJSeHcc89ly5YtbN++vc7jfPPNN1Vf2ikpKaSkHJ5U9M0332TQoEGkp6ezevXqWicK9DVnzhx+/vOf065dO6Kjo7nsssuqpjzv06cPaWlpQPVpzn0NHz6ciRMn8vLLL1NeXg7Ap59+ymuvvUZaWhpDhgwhPz+f9evXN/AsHT2rWYBTqwgJh34XB/Z9Op0M182GV8c44zB+9a7TTGVMS1RPDSCQLr30Uu666y6WLl1KcXFxVY1g2rRp7Ny5kyVLlhAeHk7v3r1rnZrcV221jqysLB5//HEWLVpEXFwcEydO9Huc+ubgq5ziHJxpzmtrhnrhhRdYsGABH374IWlpaSxfvhxV5dlnn+X888+vVvbrr7+uN5ZjZTWLigpY9S6ceA5ExQf+/RL6OgkjMhZeuxRyW9aUAMZ4LTo6mjPPPJPrr7++Wsf2vn376Ny5M+Hh4Xz11Vds2lT/FYojR45k2rRpAKxatYqVK1cCzhTn7dq1o0OHDmzfvp2PPvqoap+6phEfOXIk7733HkVFRRQWFvLuu+9yxhlnNPgzbdy4kSFDhjBp0iQ6duzI5s2bOf/883n++ecpLS0F4Mcff6SwsDBgU5lbstiXA6WFznTkTSWuF1z3EUR2cGasrShvuvc2phWYMGECK1asYPz48VXbrrrqKhYvXkxGRgbTpk2jX7/6Jwq99dZbOXDgACkpKTz66KNkZmYCzsp36enpDBw4kOuvv77aFOc333wzo0ePrurgrjRo0CAmTpxIZmYmQ4YM4cYbbyQ9Pb3Bn+eee+7h1FNPJTk5mZEjR5KamsqNN97IgAEDGDRoEMnJydxyyy2UlZWRkpJCWFgYqampjdrBbVOUA5SXOp3cTb3i3fdvw8wbnOVZU8f7L29MkLMpyoPb8UxRbjULcGaM9WJp1IGXOSvxffmwrbZnjAlqliy8FBIC5z7oNIUtmux1NMYYUydLFl7rezb0GQXfPg4lBV5HY8xxaylN2y3N8f5cLFl4TcSpXRTlw7xnvY7GmOMSGRlJfn6+JYwgo6rk5+cTGRl5zMewcRbBoMcgGHApfPccDL4RYrp4HZExx6Rnz57k5uayc6et5RJsIiMj6dnz2CdJtWQRLM65H9a+D988Chc94XU0xhyT8PBw+vTp43UYJgCsGSpYJPSF066FJa/Y7LTGmKBjySKYjLoXQiPgq4e9jsQYY6qxZBFMYrrC0FuduarylnkdjTHGVLFkEWyG/we0jYPPH/I6EmNMEDtUVkHWrkK+XreDr9ftCPj7WQd3sInsAGfcDZ/+EX76Gk440+OAjDFe2VdcyubdRWzKL2LT7kJy8ovIcZ9v3VdMhXuFcnKP9px5SgDW4vFhySIYDb4R5j8Pnz0AN33ljPQ2xrQ4FRXKtoISNuUXOUlhd6HP4yL2FpVWK98xOoKk+CgG944jKaEnveKjSEqIoldCVMBjtWQRjMIj4aw/wL9vgzXvQXITzohrjGlUJaXlVbWDnN2VNYNCNu0uInd3MYfKK6rKhoUIPeLakhQfxUWndqNXQhRJ8e3olRBFYnwU0W28+8q2ZBGsUsc7I7q//DP0v8SZ7NAYE3RUlT1FpWzKL6xqIsrZXUSO23S0vaD6JKHRbcJIio/ilC4x/GxAF5Lio+jlJoRuHSIJCw3OlgRLFsEqJBTOfQCmj4elr8HgG7yOyJhWq6y8gry9JU4yqNF3sHl3EfsPllUr37V9JEnxUZxxUqeqpqKk+Ch6JbQjLiq8znW/g5kli2B28gWQOBT+769OTSOindcRGdNiFR4s82kqKqzWbLRlTzFlFYfnu4oICyHRbS7K7BNPUnxlMnCaiyLDQz38JIFhySKYicDPHoIp58P8/4WR93gdkTHNlqqyc/9BNlU1ERWR4zYd5ewuYteBQ9XKx0aF0ys+ilN7dODilG70im9XVUPo2j6SkJDmVzs4HgFNFiJyAfA3IBT4h6o+UuP1u4AbgTJgJ3C9qm5yX7sW+JNb9P+p6quBjDVoJQ2FUy6Euc9Axg1Ns064Mc3UobIKcvc4iaDqktN8p6aQs7uIktLDnckhAt06tKVXQhQ/G9CFRJ++g8T4KDq0tX5CXwFLFiISCjwH/AzIBRaJyCxVXeNTbBmQoapFInIr8ChwhYjEAw8AGYACS9x99wQq3qB2zv3w/Onw7RNwvk0FYlq3fcWlVZ3HVZeZuk1GvmMPANqGh1b1FYw8qVO1voMesW2JCAvOzuRgFMiaRSawQVV/AhCRGcBYoCpZqOpXPuXnA1e7j88HPlPV3e6+nwEXANMDGG/w6twfUifAwpdgyC0Qm+R1RMYETEWFsrWgxO1ELqxxyWkR+4prjj1oQ1J82yP6DpISougU3aZZdiYHo0Amix7AZp/nucCQesrfAHxUz749au4gIjcDNwMkJbXwL9Azfw/fvw1f/QV+/rzX0RhzXEpKy30uLz3cd1DX2IOecW1JjI/iklSn7yCxMiHER9HOw7EHrUkgz3Jt6bzW5bNE5GqcJqdRR7Ovqr4EvASQkZHRspfmik2EzJucBZJO/w10GeB1RMbUSVXZXXiozr6DmmMPYtqEkZRweOxBZd9BUnxwjz1oTQKZLHKBRJ/nPYG8moVE5Fzgj8AoVT3os++ZNfb9OiBRNidn/Jcz5uKLSXDlDK+jMa1c5diDTW4CyMmvnMPISRAHaht7kFB97EGvhHYkxUc127EHrUkgk8Ui4CQR6QNsAcYDV/oWEJF04EXgAlX1nTbxE+B/RCTOfX4e8PsAxto8RMXDiN86yWLTd9BrmNcRmRbuwMGyOvsOtuwtpryWsQe9EtoxxO0/6OXOW9QzrmWOPWhNApYsVLVMRO7A+eIPBaao6moRmQQsVtVZwGNANPCW+19FjqqOUdXdIvJnnIQDMKmys7vVG3IrLHgJPn8Arv/EGYthzDFSVXbsP3h4mgqfvoOc/CLyC6uPPYiLCicpPorUxFjGpHZ3OpTdhNAlpvWNPWhNRLVlNPVnZGTo4sWLvQ6jaSyeAh/8J4yfDv0u9DoaE+QOlpWTu6e4WlNRZd9BbWMPuse2PXxFUXy7aiOTbexByyMiS1Q1w185u4ygOUr/Fcz7O3zxEJx8vjOPlGnV9hWVVvUdODUEZxzC5t3F5O0rRmuMPeiVcHjsgXOZqZMUbOyBqYsli+YoNNwZqPfWtbBiOqRf7X8f06yVV6174DOJnVtTyNld+9iDXglRDOkTX3WZaWXtwMYemGNhyaK5GjAWug9yxl0kj3PWwDDNWvGhcjbvqbzMtLBqAZyc/CJy99Q+9iApoR1pibHV+g4S42zsgWl89hvVXInAuQ/Ca2Ng0cvO2AsT1FSV/MJDNfoODl9ptGN/7WMP+nWL4byBXX36EWzsgWl6liyasxNGQd+znTmj0n8FbWO9jqjVKy2vIG9vcbVFcJxFcYrJyS+k8FB5tfLdOkSSGB/FqJOr9x30io8i1sYemCBiyaK5O/dBeHEkzP2bs1iSaRIHy8pZnVfA8py9bNh5oKrvoLaxB5XzFQ3pE1/Vd5AUb2MPTPNiyaK565bq9FnMfx4yb4b23byOqMVRVXL3FLNs816W5exhWc5e1uQVVPUhxEWFV/UdjEnt7vQduH0INvbAtBSWLFqCs/8Ia95zVtS75Gmvo2n2Cg+WsTJ3H8s2O4lhWc5edh1w+hMiw0NI6RnLdSN6k54YR3pSLF3a28UFpuWzZNESxJ8AGdfDoskw7HboeJLXETUbFRXKT7sKnRrDZicxrNtWULUmwgkd2zHy5I6kJ8WRnhjLKV1jCLeOZdMKWbJoKUbeA8umwZd/hl++5nU0QWtv0aGqpLB8816W5+yhoMSZ8C4mMoz0pDh+NuAk0pNiSesZS1y7CI8jNiY4WLJoKaI7w+l3OE1RW5ZAj9O8jshzZeUV/LBtf1Vfw/Kcvfy0qxBwprU4pWt7Lk7tTnpiLOlJcZzQsZ31LxhTB0sWLcmwO2DRP+CzB+Da91vdJIM7CkpYmrO3qq/h+9x9FJc6l6p2jI4gPSmOcRk9SU+MI6VnBxu4ZsxRsL+WliSyPYz8HXx8L2z8Ak481+uIAqak1Ll0tbKvYXnOXrbsLQYgPFQY2L0D4zMTq/oaesa1tTELxhwHSxYtTcZ1MP85+PxBOOFsCGn+nbGqyubdxYevTtq8lzV5+ygtd3qhe8S2JT0plutH9CE9KZYB3drb+AVjGpkli5YmrA2c9Sd492ZY/Q6cOs7riI7agYNlrNy8t9q4hsp1FdqGh5Ka2IEbzziBtMRY0hNj6WyXrhoTcJYsWqJTL4d5zzhXRvUfA2HBe0VPRYWycecBt8bgJIYft++vunS1b6d2nNWvM+lJsaQnxnFyl2ibE8kYD1iyaIlCQpxpQKaNgyWvwJCbPQ7osD2Fh1heWWNw+xr2u2s1t3cvXb0guSvpSXGk9YylQ5QttmNMMLBk0VKdeC70GgHfPAppV0Kb6CYPobS8gnXb9lc1JS3bvJcsn0tX+3Vtz5i07k4ndFIsfRLs0lVjgpUli5aqcgrzyefCd8/BmfcG/C237SvxGQm9h++37KtasrNTTBvSE2P5ZUYi6UmxnNrDLl01pjmxv9aWLHEw9L/E6b8YfAO069hohy4pLWfVln3V+hq27isBICI0hOQe7blqSC9nJHRiLD1i7dJVY5ozSxYt3dn3ww8fwjePwei/HtMhVJWc3UXupHp73EtXCyhze6ET49syuHe80wmdFEf/bjG0CbNLV41pSSxZtHSdTnbW6F40GYbeCnG9/e6yv6TUmXXVp69ht3vpalREKKk9Y7l55AlOJ3RiLJ1i2gT4QxhjvGbJojU48/ew8k346n/gspfqLbpqyz4uf+G7qmkyTuwczTn9Old1Qp/cJYZQ64Q2ptWxZNEatO8OQ37trKZ3+m+g66l1Fn3h/zYSFiq8+qtM0hJj6dDWLl01xoCNbmotRvzWmTvq84fqLLJlbzEfrdrGhMwkRp3cyRKFMaaKJYvWom0cjLgLNnwG2XNqLfLavGwArj29d9PFZYxpFixZtCZDboGY7s4U5qrVXio8WMb0hTlcMLArPWLbehSgMSZYWbJoTcLbwlm/hy2LYe371V6auTSXgpIyrh/Rx6PgjDHBzJJFa5N6JXQ8Gb6YBOXOnEwVFcrUudmkJsYyKCnW4wCNMcHIkkVrExoG59wP+eth+TQAvlq3g6xdhdwwoo+NsjbG1MqSRWvU72LoORi+fgQOFTF5ThbdOkQyOrmr15EZY4KUJYvWSATOfQj257Hji2eYtzGfa4b1JtzWiTDG1KHB3w4iEikiN4jIb0QkIZBBmSbQezicdB7Ri56lS3gxV2YmeR2RMSaIHc2/kn/DGfFdArwXmHBMU9o97PdElhfyeLcvbZEhY0y96kwWIvKGiPT12RQPTAOmA3GBDswE3ms/RfNuxXCG58+EfVu8DscYE8Tqq1n8CfiziDwuIh2Ax4FZwKfAgw05uIhcICLrRGSDiNxXy+sjRWSpiJSJyLgarz0qIqtFZK2IPCN2mU6jKikt5/X5m5if9GtCqICv/+J1SMaYIFZnslDVn1T1Spwmp38BmcDPVPV0VX3b34FFJBR4DhgNDAAmiMiAGsVygInAGzX2PR0YDqQAycBgYFQDP5NpgFkr8th14BCXnjUMMm5wLqPduc7rsIwxQarOWWdFJA64EigFfglcCnwiIk+r6gcNOHYmsEFVf3KPNwMYC6ypLKCq2e5rFTX2VSASiAAECAe2N+wjGX9UlSlzsujXNYbT+yZAt7th2evOQL3x07wOL/iowqFCOFgAB/e7N5/H4W2daVTad4OYbhBm63uYlqe+KcrfA/4JRAH/VNWxIvIW8DsRuVlVx/g5dg9gs8/zXGBIQ4JS1e9E5CtgK06y+Luqrq1ZTkRuBm4GSEqyq3ka6ruN+fywbT+P/iLFGYTXriMMvxO+ehg2L4TETK9DbBwV5T5f7vtr/6Kvd5vPa6jft6sSlVA9ebTvfuR92zjnEmZjmon6kkUCTvNQW+AaAFUtBh4SkW4NOHZtfwkN+osTkROB/kBPd9NnIjJSVb+pdjDVl4CXADIyMo7ir7l1mzwni4R2EYxJ635449DbYOHL8PmDMPFDb7/IykuP8ku9xrYS93lpYcPeLyIG2tS4xXSFNu2P3N4mpsb2aDhUBPvzoGAr7N8KBXmH7/OWQeHOI98ztI2bTOpJKjHdICyicc+tMceovmRxP/AZUA5U65xW1a0NOHYukOjzvCeQ18C4fg7MV9UDACLyETAU+KbevYxfWbsK+eKHHdx5zklEhvusk90mGkb9DmbfDes/g5PPO7oDq0LZwWP8kq/xWlmJ//eTkCO/uNvGQWxSLV/oNb/ofV6LiIaQRhiM2KVmd5yPskNwYJubTPJq3G+FLUud+9o+d1THGkml5n03q6WYJlFnslDVd4B3juPYi4CTRKQPsAUYj9MH0hA5wE0i8hecGsoo4OnjiMW4ps7NIiI0hKuH1tJsN+ha+O45p3ZRfrDGf+p1NdP4bK8o9R9ASJjPl7V7H92TSy0LAAAYGklEQVQVEk6q4z/3GGfRpprbw6OazxdkWISTxGLraSpVheI9bo2ktqSSB1uWQNGuWo4fWaNmUktSie5qtRRzXAK2rKqqlonIHcAnQCgwRVVXi8gkYLGqzhKRwcC7OOM2LhGRh1R1IPA2cDbwPU7T1ceq+n7t72Qaal9RKW8tzuWS1O50jok8skBYhDPJ4NvXwb+urvFa5JH/occm+v/PvWYCCGvTfL7km5IIRMU7ty4D6y5XdhD2bzuyuasyyWxZDGu3Osm+pnad6uhD8UkqkbH28zG1Cuga3Ko6G5hdY9v9Po8XcbhfwrdMOXBLIGNrjWYsyqG4tJwb6luzYuDPIaFv9WaeiGj7rzRYhLWBuF7OrS6VtZQjkol7v28L5C6Covxajt+2AX0pXSHURvy3NvUmC3esxJ2q+lQTxWMCpKy8glfnZTPshAQGdG9fd0ER6JbadIGZxudbS+maXHe5soO1NHv5JJXNC5378kM138CppdTVh1KZVCI7WC2lBak3WahquYiMBSxZNHMfr95G3r4SHhpbz5eHaV3C2kBcb+dWF1Uo2l17H0rBVti3GTYvgOLdR+4bHtWAvpQuVktpJhrSDDVXRP6OM4q76lpEVV0asKhMo5s8J4veCVGc06+z16GY5kQE2iU4t66n1l2utMSphdTsQ6lMKpvnO30ttdVSojv7Typt2lstxWMNSRanu/eTfLYpTge0aQaW5uxhWc5eHhozkJAQ+4MzARAeCfF9nFtdVJ1+krr6UvZsgpzvnP6WI47frv7xKJVXfIUGtBu2VfN7ZlX1rKYIxATOlDlZxESGMe60I64lMKbpVM4W0K4jdEupu1xpsU/NpJaksuk75/6IS7UbUkvp7lyKbY6a32Thzjj7ADDS3fR/wCRV3RfIwEzj2LK3mI9WbeOGEX1o18b+6zLNQHhbiD/BudWlosKppdTVl7InGzbNg5K9R+4bEV13c1flfbvOVkupoSFnYwqwCmcyQYBfAVOBywIVlGk8r83LRlW5Zlg9l1oa09yEhEB0J+dW39V7h4p8+lJqSSqb5rq1lLLq+0mI0/leVUvpWnsTWCuqpTQkWfRV1V/4PH9IRJYHKiDTeAoPljF9YQ6jk7vRMy7K63CMaXoRUc64oYS+dZepqHBGxtc2yHF/HuRvhOxvoaSWxpSG1FKiu0BI6JH7NjMNSRbFIjJCVecAiMhwoDiwYZnGMHNpLgUlZVxf3yA8Y1q7kBCnryO6M5BWd7nKWkpdHfTZc5w5wPzWUmokl/Y9nG1togP6MY9XQ5LFr4HX3L4LgD3AtYELyTSGigpl6txsUhNjGZQU63U4xjR/Da2lFO6sfcLIgjzI3wBZ38LBWmopbdrX3dxVmVyiO3tWS/E3gjsEOEVVU0WkPYCqFjRJZOa4fLVuB1m7CnlmQjq2Iq0xTSQkBGK6OLfu6XWXO1RY94SRBVsh6xtnXIqWV99PQp1aSs3LiDudAv0uCuhH8zeCu8KdDPBNSxLNy+Q5WXTrEMno5K5eh2KMqSmiHXQ80bnVpaLcqaXU1Zeya72TVA4WQOIQb5OF6zMRuZsjR3DXMr7fBIO1WwuYtzGfey/oR3hoI6zVYIxpeiGhbrOUn3/4Dh6AQwcCHk5DksX17v3tPtsUqOciaOOlKXOyaBseypWZttSsMS1em+gm6RxvSJ/F1ao6N+CRmEaxc/9B/r08jysGJ9IhyiZoM8Y0jnrbKFS1Ani8iWIxjWDagk0cKq9g4vDeXodijGlBGtKg/amI/ELskpqgV1JazuvzN3F2v8707RTc12wbY5qXhvRZ3AW0A8pFpBhnTWxV1dYzzr2ZmLUij10HDtW/Ep4xxhyDhsw6G9MUgZjjo6pMmZNFv64xnN43wetwjDEtjN9mKHFcLSL/7T5PFJHMwIdmjsZ3G/P5Ydt+rh/exwbhGWMaXUP6LP4XGAZc6T4/ADwXsIjMMZk8J4uEdhGMSevudSjGmBaoIcliiKreDpQAqOoeICKgUZmjkrWrkC9+2MFVQ3sRGd78Z7c0xgSfhiSLUhEJxRmIh4h0AioCGpU5KlPnZhERGsLVQ20QnjEmMBqSLJ4B3gU6i8jDwBzgfwIalWmwfUWlvLU4l0tSu9M5JtLrcIwxLVRDroaaJiJLgHNwLpu9VFXXBjwy0yAzFuVQXFpul8saYwKqQYvMquoPwA8BjsUcpbLyCl6dl82wExIY0N2GvRhjAsemJG3GPl69jbx9JbYSnjEm4CxZNGOT52TROyGKc/p19joUY0wLZ8mimVqas4dlOXu5bngfQkJsEJ4xJrAsWTRTU+ZkERMZxrjTenodijGmFbBk0Qxt2VvMR6u2MSEziXZtGnSNgjHGHBdLFs3Qa/OyUVWuGdbL61CMMa2EJYtmpvBgGdMX5jA6uRs946K8DscY00pYsmhmZi7NpaCkzC6XNcY0KUsWzUhFhTJ1bjapibEMSor1OhxjTCsS0GQhIheIyDoR2SAi99Xy+kgRWSoiZSIyrsZrSSLyqYisFZE1ItI7kLE2B1+t20HWrkJuGGFrVhhjmlbAkoU7U+1zwGhgADBBRAbUKJYDTATeqOUQrwGPqWp/IBPYEahYm4vJc7Lo1iGS0cldvQ7FGNPKBLJmkQlsUNWfVPUQMAMY61tAVbNVdSU1pjx3k0qYqn7mljugqkUBjDXord1awLyN+VwzrDfhodZ6aIxpWoH81ukBbPZ5nutua4iTgb0i8o6ILBORx9yaSjUicrOILBaRxTt37myEkIPXlDlZtA0P5cpMW7PCGNP0ApksamtU1wbuGwacAdwNDAZOwGmuqn4w1ZdUNUNVMzp16nSscQa9nfsP8u/leYw7rScdosK9DscY0woFMlnkAok+z3sCeUex7zK3CasMeA8Y1MjxNRvTFmziUHkFE4f39joUY0wrFchksQg4SUT6iEgEMB6YdRT7xrlLuAKcDawJQIxBr6S0nNfnb+Lsfp3p2yna63CMMa1UwJKFWyO4A/gEWAu8qaqrRWSSiIwBEJHBIpILXA68KCKr3X3LcZqgvhCR73GatF4OVKzBbNaKPHYdOMT1w20QnjHGOwGdhU5VZwOza2y73+fxIpzmqdr2/QxICWR8wU5VmTIni35dYxh+YoLX4RhjWjG7BjOIfbcxnx+27ef64TYIzxjjLUsWQWzynCwS2kUwJq2716EYY1o5SxZBKmtXIV/8sIOrhvYiMvyIISbGGNOkLFkEqalzs4gIDeHqoTYIzxjjPUsWQWhfUSlvLc7lktTudI6J9DocY4yxZBGMZizKobi0nBtszQpjTJCwZBFkysoreHVeNsNOSGBA9/Zeh2OMMYAli6Dz8ept5O0rsZXwjDFBxZJFkJk8J4teCVGc06+z16EYY0wVSxZBZGnOHpbl7OW603sTEmKD8IwxwcOSRRCZMieLmMgwLs9I9F/YGGOakCWLILFlbzEfrdrGhMwk2rUJ6JRdxhhz1CxZBInX5mWjqlwzrJfXoRhjzBEsWQSBwoNlTF+Yw+jkbvSMi/I6HGOMOYIliyAwc2kuBSVldrmsMSZoWbLwWEWFMnVuNqmJsQxKivU6HGOMqZUlC499tW4HWbsKuWGErVlhjAleliw8NnlOFt06RDI6uavXoRhjTJ0sWXho7dYC5m3M55phvQkPtR+FMSZ42TeUhybPyaJteChXZtqaFcaY4GbJwgOl5RU8/OEa3l6SyxWDE+kQFe51SMYYUy8bKtzEduwv4Y5py1iYvZtrh/XiDxf29zokY4zxy5JFE1qYtZvb31jKgZIynr4ijUvTe3gdkjHGNIgliyagqkyek8VfPvqBpPgo/nlDJv262sJGxpjmw5JFgB04WMa9M1fy4cqtnD+wC49dnkr7SOujMMY0L5YsAmjDjv3c8s8lZO0q5L7R/bhl5Ak28M4Y0yxZsgiQD1bmce/bK2kbEcrrNw7h9L4dvQ7JGGOOmSWLRlZaXsFfZv/AlLlZDEqK5X+vOo2uHSK9DssYY46LJYtGtKOghNvfWMqi7D1MPL03f7iwPxFhNpTFGNP8WbJoJAt+yueO6cs4UFLG38anMTbNLos1xrQcliyOk6ryj2+zeOTjH+gVH8XrNwzhlK4xXodljDGNypLFcThwsIzfvb2C2d9v44KBXXns8hRi7LJYY0wLZMniGK3fvp9bXl/Cpvwi/nBhP246wy6LNca0XJYsjsH7K/K4d+ZKoiJCmXbjEIaekOB1SMYYE1CWLI5CaXkF/zN7LVPnZnNarzj+96pBdGlvl8UaY1q+gF7XKSIXiMg6EdkgIvfV8vpIEVkqImUiMq6W19uLyBYR+Xsg42yI7QUlTHhpPlPnZnPd8N7MuHmoJQpjTKsRsJqFiIQCzwE/A3KBRSIyS1XX+BTLASYCd9dxmD8D/xeoGBtq/k/53PHGMooOlfHMhHTGpHb3OiRjjGlSgaxZZAIbVPUnVT0EzADG+hZQ1WxVXQlU1NxZRE4DugCfBjBGvz5etZWr/rGA9m3DeO/24ZYojDGtUiCTRQ9gs8/zXHebXyISAjwB3OOn3M0islhEFu/cufOYA63Pi9/8RO+EKP59+3BO7mLjJ4wxrVMgk0Vt15FqA/e9DZitqpvrK6SqL6lqhqpmdOrU6agD9Gfz7iKW5exl3GmJNn7CGNOqBfJqqFwg0ed5TyCvgfsOA84QkduAaCBCRA6o6hGd5IH0wcqtAFyc0q0p39YYY4JOIJPFIuAkEekDbAHGA1c2ZEdVvarysYhMBDKaOlGAM814WmIsifFRTf3WxhgTVALWDKWqZcAdwCfAWuBNVV0tIpNEZAyAiAwWkVzgcuBFEVkdqHiO1sadB1idV8Al1qFtjDGBHZSnqrOB2TW23e/zeBFO81R9x3gFeCUA4dXrgxVbEYGLTrUmKGOMscUWaqGqzFqxhcze8bZwkTHGYMmiVj9s28/GnYXWBGWMMS5LFrV4f0UeoSHC6OSuXodijDFBwZJFDarK+yvzGH5iRxKi23gdjjHGBAVLFjWsyN3H5t3FNrbCGGN8WLKo4YMVeYSHCucPtCYoY4ypZMnCR0WF8sHKrYw6uTMd2tr0HsYYU8mShY/Fm/awraCES1KtCcoYY3xZsvDx/oo8IsNDOLd/F69DMcaYoGLJwlVWXsHs77dyTv8utGtjq80aY4wvSxau737KJ7/wEJek2EA8Y4ypyZKF6/0VeUS3CePMUxp/XQxjjGnuLFkAh8oq+HjVNs4b0IXI8FCvwzHGmKBjyQL4dv1OCkrKbC4oY4ypgyULnCao2Khwhp/Y0etQjDEmKLX6ZFF8qJzP1mxndHJXIsJa/ekwxphatfpvx4KSUs7u34VL03p4HYoxxgStVj+goEv7SJ6dkO51GMYYE9Rafc3CGGOMf5YsjDHG+GXJwhhjjF+WLIwxxvhlycIYY4xfliyMMcb4ZcnCGGOMX5YsjDHG+CWq6nUMjUJEdgKb6inSEdjVROEcDYur4YIxJrC4jkYwxgStO65equp3bYYWkyz8EZHFqprhdRw1WVwNF4wxgcV1NIIxJrC4GsKaoYwxxvhlycIYY4xfrSlZvOR1AHWwuBouGGMCi+toBGNMYHH51Wr6LIwxxhy71lSzMMYYc4wsWRhjjPGrVSQLEblARNaJyAYRua+J3ztbRL4XkeUistjdFi8in4nIevc+zt0uIvKMG+dKERnUiHFMEZEdIrLKZ9tRxyEi17rl14vItQGK60ER2eKes+UicqHPa79341onIuf7bG+0n7GIJIrIVyKyVkRWi8h/uNs9PV/1xOXZ+RKRSBFZKCIr3Jgecrf3EZEF7uf+l4hEuNvbuM83uK/39hdrI8f1iohk+ZyrNHd7k/3Ou8cMFZFlIvKB+9zT89Ugqtqib0AosBE4AYgAVgADmvD9s4GONbY9CtznPr4P+Kv7+ELgI0CAocCCRoxjJDAIWHWscQDxwE/ufZz7OC4AcT0I3F1L2QHuz68N0Mf9uYY29s8Y6AYMch/HAD+67+3p+aonLs/Ol/uZo93H4cAC9xy8CYx3t78A3Oo+vg14wX08HvhXfbEex7mqK65XgHG1lG+y33n3uHcBbwAfuM89PV8NubWGmkUmsEFVf1LVQ8AMYKzHMY0FXnUfvwpc6rP9NXXMB2JFpFtjvKGqfgPsPs44zgc+U9XdqroH+Ay4IABx1WUsMENVD6pqFrAB5+fbqD9jVd2qqkvdx/uBtUAPPD5f9cRVl4CfL/czH3Cfhrs3Bc4G3na31zxXlefwbeAcEZF6Yj0m9cRVlyb7nReRnsBFwD/c54LH56shWkOy6AFs9nmeS/1/YI1NgU9FZImI3Oxu66KqW8H5AgA6u9ubOtajjaMp47vDbQ6YUtnc40VcbrU/Hec/06A5XzXiAg/Pl9ukshzYgfNluhHYq6pltRy/6r3d1/cBCY0dU21xqWrluXrYPVdPiUibmnHVeP9A/AyfBn4HVLjPEwiC8+VPa0gWUsu2prxeeLiqDgJGA7eLyMh6ynoda6W64miq+J4H+gJpwFbgCS/iEpFoYCbwW1UtqK+ox3F5er5UtVxV04CeOP/d9q/n+E12rmrGJSLJwO+BfsBgnKale5syLhG5GNihqkt8N9fzHl7/LVZpDckiF0j0ed4TyGuqN1fVPPd+B/Auzh/T9srmJfd+h0exHm0cTRKfqm53/9ArgJc5XL1usrhEJBznC3maqr7jbvb8fNUWVzCcLzeOvcDXOG3+sSISVsvxq97bfb0DTjNkwH63fOK6wG3KU1U9CEyl6c/VcGCMiGTjNP+djVPTCJrzVadAdogEww0Iw+mU6sPhzryBTfTe7YAYn8fzcNo7H6N6R+mj7uOLqN7JtrCR4+lN9Y7ko4oD5z+xLJyOvjj3cXwA4urm8/g/cdpmAQZSvVPvJ5zO2kb9Gbuf+zXg6RrbPT1f9cTl2fkCOgGx7uO2wLfAxcBbVO+wvc19fDvVO2zfrC/W4zhXdcXVzedcPg084sXvvHvsMzncwe3p+WpQvIE8eLDccK50+BGnLfWPTfi+J7g/0BXA6sr3xmlz/AJY797Hu9sFeM6N83sgoxFjmY7TRFGK81/JDccSB3A9TmfaBuC6AMX1T/d9VwKzqP5l+Ec3rnXA6ED8jIEROFX6lcBy93ah1+ernrg8O19ACrDMfe9VwP0+v/sL3c/9FtDG3R7pPt/gvn6Cv1gbOa4v3XO1Cnidw1dMNdnvvM9xz+RwsvD0fDXkZtN9GGOM8as19FkYY4w5TpYsjDHG+GXJwhhjjF+WLIwxxvhlycIYY4xfYf6LGNMyichfgE+AWKCfqj7icUh1EpEHgQOq+rjXsZjWyWoWpjUbgjO30iicQVvGmDpYsjCtjog8JiIrceYH+g64EXheRO6vpWwnEZkpIovc23B3+zOV5UXkfBH5RkRCROQSd92BZSLyuYh0ccs8KCKvisin4qxxcpmIPCrOWicfu9N4VK5/8ld3LYaFInJiLTH1dfdZIiLfikg/d/vlIrLKXcPhm0CdP9NKBXrUn93sFow3nDmBnsWZunpuPeXeAEa4j5OAte7jKJxR+WfhjKDt626P4/Da9jcCT7iPHwTmuO+XChThjrrFmTPsUvdxNodH+l/D4RG+D+KuWYEzevwk9/EQ4Ev38fdAD/dxrNfn2G4t62Z9Fqa1SseZLqMfsKaecucCA5wlBABoLyIxqrpfRG4CvgH+U1U3uq/3BP7lTjQYgTOXUKWPVLVURL7HmaPpY3f79zjzY1Wa7nP/lG8w7oyzpwNv+cRUOc32XOAVEXkTeAdjGpElC9OqiLOM5is4X+q7cGoI4q57MExVi2vsElLHdoBTgXygu8+2Z4EnVXWWiJyJUyOodBBAVStEpFRVK+faqaD636LW8bgynr3qTL1djar+WkSG4EyKt1xE0lQ1v5a4jTlq1mdhWhVVXe5+0VYuSfolcL6qptWRED4F7qh8IofXbO4F/BdODWW0+yUNzhTSW9zH1x5jmFf43H9XI/4CIEtELnfjEBFJdR/3VdUFqno/TiL0ncLamONiycK0OiLSCdijzvoP/VS1vmaoO4EMd2W1NcCv3WUtJ+P0IeThzJT7DxGJxKlJvCUi3+J8YR+LNiKyAPgPnCnHa7oKuEFEKmczrlwS9TG3w3wVTvPYimN8f2OOYLPOGhNE3EVxMlT1WBONMQFhNQtjjDF+Wc3CGGOMX1azMMYY45clC2OMMX5ZsjDGGOOXJQtjjDF+WbIwxhjj1/8HpF8kQlzmUmQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(num_exs,spam_train_err,label=\"Training set\")\n",
    "plt.plot(num_exs,spam_val_err,label=\"Validation set\")\n",
    "plt.xlabel(\"# examples\")\n",
    "plt.ylabel(\"error %\")\n",
    "plt.title(\"Spam\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exs = [100, 200, 500, 1000, 2000, 5000] #number of examples\n",
    "cifar10_train_err = []\n",
    "cifar10_val_err = []\n",
    "for num_ex in num_exs:\n",
    "    train_err, val_err = \\\n",
    "    train(cifar10_xtrain, cifar10_ytrain, cifar10_xval, cifar10_yval, \\\n",
    "          num_ex, \"linear\")\n",
    "    cifar10_train_err.append(train_err)\n",
    "    cifar10_val_err.append(val_err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xu8VWW97/HPl5uoIHdTAQONbQItFrQE25jiJRU1cRsVlDuplKNpVh4tapcZ+3Ryp6llbO9a7aMSahoZirrV46VUFoYoGIJKxyWESIqKqCz4nT/GWMPJZK4razAXa33fr9d8zTGe+Yxn/J7JYvzmuD1DEYGZmRlAp3IHYGZmbYeTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4K1mFI+oKkaklvS1ot6W5Jh0i6UNL/KagXkjak9d6W9EZROx9J6/yiqLxL0bI1ki6WVO//M0kDJf0hjSckDSr6vLukX0l6M63zjdb6PsxKcVKwDkHSucDlwP8GPgTsC/wnMKmeRUZFRI/01bvos1OBfwBTJXUtseyIiOgBHAH8a1q/PluAecDkej7/d2BIGu+ngO9JOqqB9sy2i5OCtXuSegEzgbMi4ncRsSEiNkXEHyLi/Ga2JZIN/XcBAcfXVzcingf+BFQ2UGd1RFwJLKynypeAmRHxRkQ8C9wATGtOzGbN4aRgHcEngO7AHa3Q1gSSPY3fAreSbLRLknQgMB5Y0ZIVSRoA7Ak8XVD8NDCiJe2ZNYWTgnUE/YDXIqK2Gcs8JemN9FV47uBU4I8RsR64GTheUr+iZRdL2gAsBe4Drm5h3D3S9/UFZeuBni1sz6xRTgrWEawD+kvq0oxlxkRE7/R1DoCk3YHPADeldR4FVgNTi5atINlwf4FkL2W3dPkJBSevn6Zxb6fvexSU7QG81Yx+mDWLk4J1BH8G3gVO2s52PkPy6/0aSX8nSQh7UeIQUkRsiYhbgGrg+2nZQwUnr0c1trKIWAusBQrrjgKWbGc/zOrlpGDtXnqo5wJglqSTJO0mqaukiZJ+2oymTgWuBT5GcvK4EjgUqErPH5TyE+CM9PxASZK6A7uks7tI2qXg498AP5DUW9Jw4CvAr5oRs1mzOClYhxARlwLnkvxqXwu8DJwN3NmU5SXtS3KS+fKI+HvB60ngfuq57DQiFpHsqZxXT7tdgI1A3b0QK4ANBVV+kMb6MvAA8JOIuL8pMZu1hPyQHTMzq+M9BTMzyzgpmJlZxknBzMwyTgpmZpZpzs08bUL//v1jyJAh5Q7DzGynsnDhwtciot5Lo+vsdElhyJAhVFdXlzsMM7OdiqS/NaWeDx+ZmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZpmOkxTWPg8P/C/Y9G65IzEza7M6TlJ4/m54+GK4ajysfLTc0ZiZtUkdJymM/wb86x2wpRZ+dTzM/TpsfH37233vbVj3Aqx8DJ69Hf76x6TMzGwntNMNc7Fd9j8CzvwzPPQT+PMsWHYPHPdTGH4SSB/Ui4D33oK318Bbq+GtNfD23+Gt9FVY/n6JZ6h37gZDD4MDJiavPfbZcX00M9sOO92T16qqqqJVxj5a/XSyt7D6adhvAuzaN93gpxv+Te9su0yXXaHnh6Dn3tDjQ9Bzr+TVY68Ppje8Bs/fk+wxvP5Sstw+o+GA45IE8aGRWycgM7MdQNLCiKhqtF6HTQoAm2vhiavgz7+ErrulG/h0o9/zQ1tv7HvuBbvs0fQNegSsXQbL5sGyu6FmARDQa3C6B3EcfHg8dOnWOn0xM2uAk0Jb8/aryR7EsrvhhQehdmOSZD5yVJIghh0Fu/Ypd5Rm1k41NSl0rHMK5dRjTxjzpeT1/jvw0v9N9yLugSW/g05d4MP//MFhpj5Dyh2xmXVA3lMoty1b4JWFHxxmWvtcUr7n8PQw0/HJOYlOHedCMTNrfT58tLP6x4tJclh2N/ztTxCbk/Mc/3Rsshex32HQdddyR2lmOxknhfbgnX/AivuTvYjl9yeXv3bdLbm09oCJMOwY6NHo0/XMzNrGOQVJxwI/BzoD10XERUWfXwYcns7uBuwZEb3zjGmnsltfqPhc8qp9L7kTu24v4q93AYLBYz84zNR/WMNXR0VAbIEtm5P32JxOb04+y6a3fDC9Jf1sq7qFbRTXrWt3y7ZtxZYPyreJYUuJupsL2iq13JYPYu/UGdQpOTfTqTOoczrdKZ1O57PpziXKOxUs17lE3cLyTkXtdUnXX1y3VEydfVmytVm57SlI6gw8D3wKqAEWAFMjYmk99b8OjI6IrzTUbofaU6hPBPx9cZog5iX3WgB0751sbOrbIMeW8sa93fTBxrVwAyy2TjZbapPpNk2tnKg6NdzeNkmyvvaKElyj7TUnGTc1kTYQk5Npi7WFPYWxwIqIeDENaDYwCSiZFICpwA9zjKf9kGDvUclrwgxYX5OepP5r+h+n09YblK02oJ22/g+r4uniugW/jLep29A6it4LNyaqZ+Ne9wu63tiauUHYsuWDBFG4N5NN19ZTXpdYtmydZIrrbNXGlqL2aksnqtxier+edTfUXnHbm9v+DwcVJ5Ydkfia2l6JxNdgEm9Ke0XlPfdOjiDkKM+kMBB4uWC+BhhXqqKkDwNDgQfq+Xw6MB1g3333bd0o24Neg2Ds6eWOou3p1Ak6+ebAZik8jNjsJNPcxFecSJubjJsaU4n26tZd+14j7RX1pb722EHnZo//GRx0Wq6ryDMplPpZV983NwW4LaL0Pn9EXANcA8nho9YJz8y2IUHnLvgWpmba6lxZY4mvqYm0RKLauyL3ruT5L18DDC6YHwSsqqfuFOCsHGMxM8tPp05AJ+jctdyRbLc874haAAyTNFRSN5IN/9ziSpIOAPoAf84xFjMza4LckkJE1AJnA/OB54A5EbFE0kxJJxZUnQrMjp3thgkzs3Yo1wOHETEPmFdUdkHR/IV5xmBmZk3nAXXMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmmVyTgqRjJS2TtELSjHrqfE7SUklLJN2cZzxmZtaw3J7RLKkzMAv4FFADLJA0NyKWFtQZBnwXGB8Rr0vaM694zMyscXnuKYwFVkTEixHxPjAbmFRU53RgVkS8DhARr+YYj5mZNSLPpDAQeLlgviYtK/RPwD9JekzS45KOLdWQpOmSqiVVr127Nqdwzcwsz6SgEmVRNN8FGAZMAKYC10nqvc1CEddERFVEVA0YMKDVAzUzs0SeSaEGGFwwPwhYVaLO7yNiU0S8BCwjSRJmZlYGeSaFBcAwSUMldQOmAHOL6twJHA4gqT/J4aQXc4zJzMwakFtSiIha4GxgPvAcMCcilkiaKenEtNp8YJ2kpcCDwPkRsS6vmMzMrGGKKD7M37ZVVVVFdXV1ucMwM9upSFoYEVWN1fMdzWZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWyTUpSDpW0jJJKyTNKPH5NElrJS1KX6flGY+ZmTWsS14NS+oMzAI+BdQACyTNjYilRVV/GxFn5xWHmZk1XZ57CmOBFRHxYkS8D8wGJuW4PjMz2055JoWBwMsF8zVpWbHPSFos6TZJg0s1JGm6pGpJ1WvXrs0jVjMzI9+koBJlUTT/B2BIRFQA9wO/LtVQRFwTEVURUTVgwIBWDtPMzOrkmRRqgMJf/oOAVYUVImJdRLyXzl4LfDzHeMzMrBF5JoUFwDBJQyV1A6YAcwsrSNq7YPZE4Lkc4zEzs0bkdvVRRNRKOhuYD3QGboiIJZJmAtURMRc4R9KJQC3wD2BaXvGYmVnjFFF8mL9tq6qqiurq6nKHYWa2U5G0MCKqGqvnO5rNzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmaZJo99JKk78EVgN+DmiFiXW1RmZlYWzRkQ7+fAU8C7wJ3AJ3OJyMzatE2bNlFTU8O7775b7lCshO7duzNo0CC6du3aouXrTQqSbgZ+EBEvpEV9gZvS6W+0aG1mttOrqamhZ8+eDBkyBKnUs7SsXCKCdevWUVNTw9ChQ1vURkPnFL4P/LukSyT1Ai4heR7CvcCFLVqbme303n33Xfr16+eE0AZJol+/ftu1F1fvnkJEvAh8QdIhwG+BPwKfiojNLV6bmbULTght1/b+29S7pyCpj6SzgOHA54D1wHxJJ2zXGs3MtsO6deuorKyksrKSvfbai4EDB2bz77//fpPa+PKXv8yyZcsarDNr1ixuuummBuvk4YEHHuDxxx/f4eut09CJ5juB/yK52ui/ImKSpFuBb0uaHhEn7pAIzcwK9OvXj0WLFgFw4YUX0qNHD84777yt6kQEEUGnTqV/9954442Nruess87a/mBb4IEHHqB///4cfPDBZVl/Q+cU+gE3k5xcHggQERsj4kfA/2hK45KOlbRM0gpJMxqoN1lSSGr0qUBmZqWsWLGCkSNHcsYZZzBmzBhWr17N9OnTqaqqYsSIEcycOTOre8ghh7Bo0SJqa2vp3bs3M2bMYNSoUXziE5/g1VdfBeD73/8+l19+eVZ/xowZjB07lgMOOIA//elPAGzYsIHPfOYzjBo1iqlTp1JVVZUlrELnn38+w4cPp6Kigu985zsArFmzhpNPPpmqqirGjh3L448/zgsvvMB1113HxRdfTGVlZbaeHamhPYULgPuAzcBWG/SIWN1Yw5I6A7OATwE1wAJJcyNiaVG9nsA5wBPNC93Myu1Hf1jC0lVvtmqbw/fZgx9+ekSLll26dCk33ngjV111FQAXXXQRffv2pba2lsMPP5zJkyczfPjwrZZZv349hx12GBdddBHnnnsuN9xwAzNmbPsbNiJ48sknmTt3LjNnzuSee+7hiiuuYK+99uL222/n6aefZsyYMdsst2bNGubNm8eSJUuQxBtvvAHAOeecw7e//W0OPvhgVq5cyQknnMCzzz7LaaedRv/+/fnmN7/Zou9gezV0ovl3wO+2o+2xwIr0hDWSZgOTgKVF9f4d+ClwHmZm22H//ffnoIMOyuZvueUWrr/+empra1m1ahVLly7dJinsuuuuTJw4EYCPf/zjPPLIIyXbPvnkk7M6K1euBODRRx/NfvmPGjWKESO2TWZ9+/alU6dOnH766Rx//PGccEJyWvb+++/f6rzG66+/zsaNG1vY89bTnJvXmmsg8HLBfA0wrrCCpNHA4Ii4S1K9SUHSdGA6wL777ptDqGbWEi39RZ+X3XffPZtevnw5P//5z3nyySfp3bs3p5xySslLNbt165ZNd+7cmdra2pJt77LLLtvUiYhGY+ratSvV1dXcd999zJ49myuvvJJ777032/MoXH9bkOfYR6Wui8q+QUmdgMuA/9lYQxFxTURURUTVgAEDWjFEM2uv3nzzTXr27Mkee+zB6tWrmT9/fquv45BDDmHOnDkAPPPMMyxdWnwgBN566y3efPNNTjjhBC677DL+8pe/AHDUUUcxa9asrF7duYiePXvy1ltvtXqsTdVgUpDUWdK3Wth2DTC4YH4QsKpgvicwEnhI0krgYGCuTzabWWsYM2YMw4cPZ+TIkZx++umMHz++1dfx9a9/nVdeeYWKigp+9rOfMXLkSHr16rVVnfXr13P88cczatQojjjiCC699FIgueT1scceo6KiguHDh3PttdcCMGnSJObMmcPo0aPLcqJZje3+SHooIiY0u2GpC/A8cCTwCrAA+EJELKlvPcB5EVHdULtVVVVRXd1gFTPL0XPPPceBBx5Y7jDahNraWmpra+nevTvLly/n6KOPZvny5XTpkueR+caV+jeStDAiGv3R3ZTIH5P0S5K7mjfUFUbEUw0tFBG1ks4G5gOdgRsiYomkmUB1RMxtwrrNzNqst99+myOPPJLa2loigquvvrrsCWF7NSX6f07fZxaUBXBEYwtGxDxgXlHZBfXUndCEWMzM2ozevXuzcOHCcofRqhpNChFx+I4IxMzMyq/Rq48k9ZJ0qaTq9PWzdNRUMzNrZ5pySeoNwFskg+J9DngTaHzgEDMz2+k05ZzC/hHxmYL5H0nadnAPMzPb6TVlT2Fj+kwFACSNB8p/L7aZdUgTJkzY5ka0yy+/nK997WsNLtejRw8AVq1axeTJk+ttu7FL3i+//HLeeeedbP64447LxjPaUVauXMnNN9+cS9tNSQpnALMkrUxvMvslTRwl1cystU2dOpXZs2dvVTZ79mymTp3apOX32WcfbrvtthavvzgpzJs3j969e7e4vZYoW1JIh6I4ICJGARVARUSMjojFuURjZtaIyZMnc9ddd/Hee+8ByQZy1apVHHLIIdl9A2PGjOFjH/sYv//977dZfuXKlYwcORKAjRs3MmXKFCoqKvj85z+/1YB0Z555Zjbs9g9/+EMAfvGLX7Bq1SoOP/xwDj88uTBzyJAhvPbaawBceumljBw5kpEjR2bDbq9cuZIDDzyQ008/nREjRnD00UeXHPju1ltvZeTIkYwaNYpDDz0UgM2bN3P++edz0EEHUVFRwdVXXw3AjBkzeOSRR6isrOSyyy5rle+1ToPnFCJiS3oD2pyIaN3xcc1s53f3DPj7M63b5l4fg4kX1ftxv379GDt2LPfccw+TJk1i9uzZfP7zn0cS3bt354477mCPPfbgtdde4+CDD+bEE0+s9xGVV155JbvtthuLFy9m8eLFWw19/eMf/5i+ffuyefNmjjzySBYvXsw555zDpZdeyoMPPkj//v23amvhwoXceOONPPHEE0QE48aN47DDDqNPnz4sX76cW265hWuvvZbPfe5z3H777ZxyyilbLT9z5kzmz5/PwIEDs8NR119/Pb169WLBggW89957jB8/nqOPPpqLLrqISy65hLvuuqul33K9mnL46D5J50kaLKlv3avVIzEza6LCQ0iFh44igu9973tUVFRw1FFH8corr7BmzZp623n44YezjXNFRQUVFRXZZ3PmzGHMmDGMHj2aJUuWlBzsrtCjjz7Kv/zLv7D77rvTo0cPTj755GwY7qFDh1JZWQlsPfR2ofHjxzNt2jSuvfZaNm/eDMC9997Lb37zGyorKxk3bhzr1q1j+fLlTfyWWqYpVx99JX0vfDZdAPu1fjhmtlNp4Bd9nk466STOPfdcnnrqKTZu3Jj9wr/ppptYu3YtCxcupGvXrgwZMqTkcNmFSu1FvPTSS1xyySUsWLCAPn36MG3atEbbaWgcubphtyEZervU4aOrrrqKJ554gj/+8Y9UVlayaNEiIoIrrriCY445Zqu6Dz30UIOxbI+mnFM4JSKGFr2cEMysbHr06MGECRP4yle+stUJ5vXr17PnnnvStWtXHnzwQf72t7812M6hhx7KTTfdBMCzzz7L4sXJ6dI333yT3XffnV69erFmzRruvvvubJn6hrY+9NBDufPOO3nnnXfYsGEDd9xxB5/85Ceb3KcXXniBcePGMXPmTPr378/LL7/MMcccw5VXXsmmTZsAeP7559mwYUOuw2s35ZzCJcAnclm7mVkLTZ06lZNPPnmrK5G++MUv8ulPf5qqqioqKyv56Ec/2mAbZ555Jl/+8pepqKigsrKSsWPHAslT1EaPHs2IESPYb7/9thp2e/r06UycOJG9996bBx98MCsfM2YM06ZNy9o47bTTGD16dMlDRaWcf/75LF++nIjgyCOPZNSoUVRUVLBy5UrGjBlDRDBgwADuvPNOKioq6NKlC6NGjWLatGl861stfcLBtpoydPaPgMXA76IpjxnKmYfONisvD53d9uU9dPa5wO7AZkkbSZ6oFhGxR0uCNTOztqspo6T23BGBmJlZ+TVllFRJOkXSD9L5wZLG5h+amZntaE25T+E/SU40fyGdfxuYVX91M2vv2sDpRavH9v7bNCUpjIuIs4B30xW+DnTbrrWa2U6re/furFu3zomhDYoI1q1bR/fu3VvcRlNONG+S1JnkhjUkDQC2NKVxSccCPyd5RvN1EXFR0ednkNwUt5lkD2R6RDR826CZldWgQYOoqalh7dq15Q7FSujevTuDBg1q8fJNSQq/AO4A9pT0Y2Ay8P3GFkoTySzgU0ANsEDS3KKN/s0RcVVa/0TgUuDY5nXBzHakrl27MnTo0HKHYTlpytVHN0laCBxJcjnqSRHxXBPaHgusiIgXASTNBiYBWVIoGmRvd9K9ETMzK4+m7CkQEX8F/trMtgcCLxfM1wDjiitJOovkXohuwBGlGpI0HZgOsO+++zYzDDMza6qmnGhuqVJj1W6zJxARsyJif+A71HNYKiKuiYiqiKgaMGBAK4dpZmZ18kwKNcDggvlBwKoG6s8GTsoxHjMza0SeSWEBMEzSUEndgCnA3MIKkoYVzB4P5DtQuJmZNahJ5xRaIiJq06e2zSe5JPWGiFgiaSZQHRFzgbMlHQVsAl4HTs0rHjMza1xuSQEgIuYB84rKLiiY/kae6zczs+bJ8/CRmZntZJwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllck0Kko6VtEzSCkkzSnx+rqSlkhZL+m9JH84zHjMza1huSUFSZ2AWMBEYDkyVNLyo2l+AqoioAG4DfppXPGZm1rg89xTGAisi4sWIeB+YDUwqrBARD0bEO+ns48CgHOMxM7NG5JkUBgIvF8zXpGX1+Spwd6kPJE2XVC2peu3ata0YopmZFcozKahEWZSsKJ0CVAEXl/o8Iq6JiKqIqBowYEArhmhmZoW65Nh2DTC4YH4QsKq4kqSjgH8DDouI93KMx8zMGpHnnsICYJikoZK6AVOAuYUVJI0GrgZOjIhXc4zFzMyaILekEBG1wNnAfOA5YE5ELJE0U9KJabWLgR7ArZIWSZpbT3NmZrYD5Hn4iIiYB8wrKrugYPqoPNdvZmbN4zuazcws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCyTa1KQdKykZZJWSJpR4vNDJT0lqVbS5DxjMTOzxuWWFCR1BmYBE4HhwFRJw4uq/T9gGnBzXnGYmVnT5fmM5rHAioh4EUDSbGASsLSuQkSsTD/bkmMcZmbWRHkePhoIvFwwX5OWmZlZG5VnUlCJsmhRQ9J0SdWSqteuXbudYZmZWX3yTAo1wOCC+UHAqpY0FBHXRERVRFQNGDCgVYIzM7Nt5ZkUFgDDJA2V1A2YAszNcX1mZradcksKEVELnA3MB54D5kTEEkkzJZ0IIOkgSTXAZ4GrJS3JKx4zM2tcnlcfERHzgHlFZRcUTC8gOaxkZmZtgO9oNjOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWVyTQqSjpW0TNIKSTNKfL6LpN+mnz8haUie8ZiZWcNySwqSOgOzgInAcGCqpOFF1b4KvB4RHwEuA/4jr3jMzKxxXXJseyywIiJeBJA0G5gELC2oMwm4MJ2+DfilJEVEtHYwP/rDEpauerO1mzUz22GG77MHP/z0iFzXkefho4HAywXzNWlZyToRUQusB/oVNyRpuqRqSdVr167NKVwzM8tzT0Elyor3AJpSh4i4BrgGoKqqqkV7EXlnVzOz9iDPPYUaYHDB/CBgVX11JHUBegH/yDEmMzNrQJ5JYQEwTNJQSd2AKcDcojpzgVPT6cnAA3mcTzAzs6bJ7fBRRNRKOhuYD3QGboiIJZJmAtURMRe4HvgvSStI9hCm5BWPmZk1Ls9zCkTEPGBeUdkFBdPvAp/NMwYzM2s639FsZmYZJwUzM8s4KZiZWcZJwczMMtrZrgCVtBb4WyPV+gOv7YBw2hr3u2PpqP2Gjtv37en3hyNiQGOVdrqk0BSSqiOiqtxx7Gjud8fSUfsNHbfvO6LfPnxkZmYZJwUzM8u016RwTbkDKBP3u2PpqP2Gjtv33PvdLs8pmJlZy7TXPQUzM2sBJwUzM8u0u6Qg6VhJyyStkDSj3PFsL0k3SHpV0rMFZX0l3SdpefreJy2XpF+kfV8saUzBMqem9ZdLOrXUutoKSYMlPSjpOUlLJH0jLW/X/QaQ1F3Sk5KeTvv+o7R8qKQn0n78Nh2OHkm7pPMr0s+HFLT13bR8maRjytOjppPUWdJfJN2Vzrf7PgNIWinpGUmLJFWnZeX7W4+IdvMiGaL7BWA/oBvwNDC83HFtZ58OBcYAzxaU/RSYkU7PAP4jnT4OuJvkiXYHA0+k5X2BF9P3Pul0n3L3rYE+7w2MSad7As8Dw9t7v9OYBfRIp7sCT6R9mgNMScuvAs5Mp78GXJVOTwF+m04PT//+dwGGpv8vOpe7f430/VzgZuCudL7d9zmNeyXQv6isbH/r7W1PYSywIiJejIj3gdnApDLHtF0i4mG2fRrdJODX6fSvgZMKyn8TiceB3pL2Bo4B7ouIf0TE68B9wLH5R98yEbE6Ip5Kp98CniN5nne77jdA2oe309mu6SuAI4Db0vLivtd9J7cBR0pSWj47It6LiJeAFST/P9okSYOA44Hr0nnRzvvciLL9rbe3pDAQeLlgviYta28+FBGrIdmAAnum5fX1f6f9XtJDA6NJfjF3iH6nh1EWAa+S/Od+AXgjImrTKoX9yPqYfr4e6MfO1/fLgW8DW9L5frT/PtcJ4F5JCyVNT8vK9ree60N2ykAlyjrSNbf19X+n/F4k9QBuB74ZEW8mPwZLVy1RttP2OyI2A5WSegN3AAeWqpbMGU7oAAAEkklEQVS+7/R9l3QC8GpELJQ0oa64RNV20+ci4yNilaQ9gfsk/bWBurn3vb3tKdQAgwvmBwGryhRLntaku4yk76+m5fX1f6f7XiR1JUkIN0XE79Lidt/vQhHxBvAQybHj3pLqfsQV9iPrY/p5L5LDjTtT38cDJ0paSXLI9wiSPYf23OdMRKxK318l+REwljL+rbe3pLAAGJZetdCN5CTU3DLHlIe5QN3VBacCvy8o/1J6hcLBwPp013M+cLSkPulVDEenZW1Senz4euC5iLi04KN23W8ASQPSPQQk7QocRXJO5UFgclqtuO9138lk4IFIzjzOBaakV+oMBYYBT+6YXjRPRHw3IgZFxBCS/7MPRMQXacd9riNpd0k966ZJ/kafpZx/6+U+897aL5Kz88+THIf9t3LH0wr9uQVYDWwi+TXwVZLjp/8NLE/f+6Z1BcxK+/4MUFXQzldITrytAL5c7n410udDSHZ9FwOL0tdx7b3fabwVwF/Svj8LXJCW70eygVsB3ArskpZ3T+dXpJ/vV9DWv6XfyTJgYrn71sT+T+CDq4/afZ/TPj6dvpbUbbPK+bfuYS7MzCzT3g4fmZnZdnBSMDOzjJOCmZllnBTMzCzjpGBmZpn2dkez2TYk/YTkmu3ewEcj4qIyh1QvSRcCb0fEJeWOxTom7ylYRzCOZOykw4BHyhyLWZvmpGDtlqSLJS0GDgL+DJwGXCnpghJ1B0i6XdKC9DU+Lf9FXX1Jx0h6WFInSZ9Ox/L/i6T7JX0orXOhpF9LujcdJ/9kST9Nx8u/Jx2+o24M/f9Q8uyEJyV9pERM+6fLLJT0iKSPpuWflfSskmcuPJzX92cdVLnv6PPLrzxfJOPIXEEyBPVjDdS7GTgknd6XZIgNgN1I7jQ9nOQu2f3T8j588Izz04CfpdMXAo+m6xsFvEN6Zy3JuDYnpdMr+eDu1S/xwV28FwLnpdP/DQxLp8eRDOcAyZ2sA9Pp3uX+jv1qXy+fU7D2bjTJMBkfBZY2UO8oYHjBSKx7SOoZEW9JOh14GPhWRLyQfj4I+G06WFk34KWCtu6OiE2SniF58NM9afkzwJCCercUvF9WGEw6Quw/A7cWxLRL+v4Y8CtJc4DfYdaKnBSsXZJUCfyKZOP9GskvfqXPKfhERGwsWqRTPeUAHwPWAfsUlF0BXBoRc9Phni8s+Ow9gIjYImlTRNSNJbOFrf/PRT3TdfG8ERGVxcFExBmSxpE8lGaRpMqIWFcibrNm8zkFa5ciYlG6Qa17lOcDwDERUVnPhv9e4Oy6mTSpIOnDwP8k2eOYmG6MIRmu+ZV0uqXPw/18wfufi+J/E3hJ0mfTOCRpVDq9f0Q8EREXkCS8wiGTzbaLk4K1W5IGAK9HxBaSS1EbOnx0DlCl5GHoS4EzCobwPi+SMe+/ClwnqTvJnsGtkh4h2TC3xC6SngC+AXyrxOdfBL4qqW4EzbpHy16cnrh+luSw1tMtXL/ZNjxKqlkZpA+UqYqIliYUs1x4T8HMzDLeUzAzs4z3FMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDL/H80sjGIc6F0eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(num_exs,cifar10_train_err,label=\"Training set\")\n",
    "plt.plot(num_exs,cifar10_val_err,label=\"Validation set\")\n",
    "plt.xlabel(\"# examples\")\n",
    "plt.ylabel(\"error %\")\n",
    "plt.title(\"CIFAR-10\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\pagebreak "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #4 Hyperparameter Tuning\n",
    "#### Here I find the best value of the hyperparameter C on the MNIST dataset by computing and plotting error values for various values of C on a validation set. I created a modified training function from earlier that's reusable for further problems. Also note that gamma is set to \"auto\" as a default value because that's what scikit-learn uses as default. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_hyper(data_xtrain,data_ytrain,data_xval,data_yval,\\\n",
    "                num_ex,kernel,C,gamma=\"auto\"):\n",
    "    data_ytrain = data_ytrain.reshape(-1,)\n",
    "    data_yval = data_yval.reshape(-1,)\n",
    "    model = svm.SVC(kernel=kernel, cache_size=4000, C=C, gamma=gamma)\n",
    "    model.fit(data_xtrain[:num_ex],data_ytrain[:num_ex])\n",
    "    return 1-accuracy_score(data_yval[:num_ex],model.predict(data_xval[:num_ex,:]))\n",
    "\n",
    "num_ex = 10000\n",
    "C = [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1]\n",
    "mnist_val_err = []\n",
    "for c in C:\n",
    "    val_err = \\\n",
    "    train_hyper(mnist_xtrain, mnist_ytrain, mnist_xval, mnist_yval, \\\n",
    "                num_ex, \"linear\", c)\n",
    "    mnist_val_err.append(val_err)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The various and best C value(s) and their associated errors is/are listed/printed above the graph below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C values tried with corresponding validation errors(C = key, error = value): \n",
      "0.01: 0.06689999999999996\n",
      "0.05: 0.07920000000000005\n",
      "0.1: 0.0837\n",
      "0.25: 0.08909999999999996\n",
      "0.5: 0.08919999999999995\n",
      "0.75: 0.08919999999999995\n",
      "1: 0.08919999999999995\n",
      "\n",
      "Best C value for MNIST is: 0.01\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VeW97/HPLwMJY0ACMp4GFSuoGCECPTiWOr7UqEUGJ1CRVms9tb29l+PpJK29tceqx+q1tQpaRFFpRY4Ttkc4TkckCCKDFsQoIRtkSpgSyPC7f+wV3MQMO8POzs7+vl+vvNh7rWctfk8S9pdnDc8yd0dERKS5UuJdgIiIJDYFiYiItIiCREREWkRBIiIiLaIgERGRFlGQiIhIiyhIRESkRRQkIs1gZoVmdsjMsmstX2VmbmY5ZvZ48Hp0xPrjzMwj3i81s+kR7+8ws0/NbJ+ZFZnZM8HytcGyfWZWZWblEe/vaIs+i9RHQSLSfJ8CU2remNnJQOdabXYBv4pmZ2Y2FbgW+Ja7dwPygP8CcPcT3b1bsPxN4Naa9+7+65Z3RaT5FCQizTcXuC7i/VTgz7XaPAGMMLOzotjfacBid/8EwN23uvsjrVKpSAwpSESa712gh5kNM7NUYBLwZK02B4BfA3dFub/rzOzHZpYX7FOk3VOQiLRMzajkXOAjYEsdbf4I/JOZXdjQjtz9SeD7wPnAfwNfmNnM1i1XpPUpSERaZi5wFTCNrx7WAsDdDwK/DL6soZ25+zx3/xbQE/guMMvMzm/NgkVam4JEpAXc/TPCJ90vAv7aQNM5QBZweZT7rXD354DVwEktrVMkltLiXYBIB3Aj0Mvd95tZnf+m3L3SzH4BPFDfTsxsGrAdeAPYT/gQ14nAstYuWKQ1aUQi0kLu/om7F0TR9Gkg1MD6PcAdwOdACfBb4GZ3f6vlVYrEjunBViIi0hIakYiISIsoSEREpEUUJCIi0iIKEhERaZGYXv5rZhcA/wGkAo+6+29qrc8gfBPXKGAnMMndC82sE+G7gfOAauBf3H1psM0o4HHCk+O9HKxr8IqB7Oxsz8nJab2OiYgkgRUrVuxw9z6NtYtZkATzBD1EeOqIImC5mS1y93URzW4Edrv7cWY2Gbib8HxFNwG4+8lm1hd4xcxOc/dq4GFgBuF5iV4GLgBeaaiWnJwcCgqiuTpTRERqmNln0bSL5aGt0cBGd9/k7oeA+UB+rTb5hGdHBVgAjDczA4bz5fTZXxC+pj7PzPoDPdz9f4JRyJ+By2LYBxERaUQsg2QgsDnifVGwrM427l4JlAK9gQ+AfDNLM7MhhA99DQ7aFzWyTwDMbIaZFZhZwfbt21uhOyIiUpdYBkldk9PVPpdRX5vZhEOiALgfeAeojHKf4YXuj7h7nrvn9enT6CE+ERFppliebC8iPIqoMQgorqdNUTBHURawKzhsdXtNIzN7B9gA7A7209A+RUSkDcVyRLIcGGpmQ4KrsCYDi2q1WUT4qXIAE4DX3d3NrIuZdQUws3OBSndf5+4hYK+ZjQ3OpVwHvBDDPoiISCNiNiIJZju9FVhM+PLf2e6+1sxmAQXuvgh4DJhrZhsJP9t6crB5X2CxmVUTflDQtRG7vpkvL/99hUau2BIRkdhKikkb8/LyXJf/iog0jZmtcPe8xtrpeSTSIbg7VdVOlTvV1VAVvK+uWRaxvLo6sq1HtI3Yrtb2Xy7jyPX+5fKvtj1yu8hlyfAfOGkfvj9+KOmpsZ3EREEiLXKwsooHX9/Ilt1ldXyIcsQH7pEfrE6Vc/hDvToiCNz5Stsjt+crQZCIn8vW4EN3RVrHLeccR3pqbP8OBYk0W0VVNbc9vZLFa7cxqFdnUlOMVDNSIv9M4SvL0lJSyEirWQapKUaKWfjPmnbG4deRy79sSx3Laq2vtay+5UduH7H+cB+s1jIOv7Z6lqfU2v7w+qC9KUWkA1GQSLNUVTs/fPYDFq/dxi8uGc60cUPiXZKIxIlm/5Umq652/s9fVvOfHxQz88ITFCIiSU5BIk3i7vxs0RoWrCjiB98aynfPOjbeJYlInClIJGruzl0vrefJdz/nO2cdw7+MHxrvkkSkHVCQSNR+99o/ePStT5n2zznMvOAEnTAWEUBBIlF68PUNPLhkI1NGD+bnlwxXiIjIYQoSadSjb27intf+weWnDuRXl52sEBGRIyhIpEFz3/2MX720notO7se/TxhBaopCRESOpCCRej1XsJmfLlzDt4b15f5Jp5IW42kWRCQx6ZNB6rTog2L+z19Wc8bQbB68aiSd0vSrIiJ106eDfMWra7Zy+zOryMs5ikeuzSMz1hP1iEhCU5DIEZZ8/AXff/p9RgzKYva00+jcSSEiIg1TkMhh72zcwXfnruDr/brz+PWj6ZahqdhEpHEKEgGgoHAXNz5RQE7vrsy9YQxZndPjXZKIJAgFifDB5hKmzVlO/6xMnpw+hl5dO8W7JBFJIAqSJLeueA/XzX6PXl3TmXfTGPp0z4h3SSKSYBQkSWzDtr1c89gyunRK5anpY+mf1TneJYlIAlKQJKnCHfu5+tFlpKYYT900lsFHdYl3SSKSoBQkSWjzrgNc9ad3qax2npo+hiHZXeNdkogkMAVJktlaWs7Vjy5j38FK5t44mqFHd493SSKS4HSjQBLZvvcgVz36Lrv2H+LJ6WM4cUBWvEsSkQ5AI5IksXv/Ia59bBmhknLmXH8auYN7xrskEekgNCJJAqVlFVw7exmbduxnzrTTOC3nqHiXJCIdiEYkHdy+g5VcP+c9Pt66lz9eM4pxx2XHuyQR6WA0IunAyg5VcePjy/mgqJSHrjqVc07oG++SRKQD0oikgzpYWcWMuQW8V7iLeyeewgUn9Y93SSLSQSlIOqCKqmq+N28lb27Ywd3fHkF+7sB4lyQiHZiCpIOprKrmB/NX8ff12/hl/olMzBsc75JEpINTkHQg1dXO/16wmpc+DPFvFw3j2m/kxLskEUkCCpIOwt35t4Vr+OvKLfzo3OO56cxj4l2SiCQJBUkH4O7c+Z/rePq9z7nl7GO59ZvHxbskEUkiCpIE5+78dvHHPP5OITeMG8KPz/86ZhbvskQkicQ0SMzsAjP72Mw2mtnMOtZnmNkzwfplZpYTLE83syfM7EMzW29m/xqxTWGwfJWZFcSy/kTwwH9t5OGln3D1mH/ipxcPU4iISJuLWZCYWSrwEHAhMByYYmbDazW7Edjt7scB9wF3B8uvBDLc/WRgFPCdmpAJnOPuue6eF6v6E8Ef//sT7vv7P/j2yEH8Mv8khYiIxEUsRySjgY3uvsndDwHzgfxabfKBJ4LXC4DxFv40dKCrmaUBnYFDwJ4Y1ppwnninkP/7ykdcPKI/v50wgpQUhYiIxEcsg2QgsDnifVGwrM427l4JlAK9CYfKfiAEfA7c4+67gm0ceM3MVpjZjPr+cjObYWYFZlawffv21uhPuzH/vc/5+aK1nDv8aO6blEuqQkRE4iiWQVLXp5tH2WY0UAUMAIYAPzKzmutZx7n7SMKHzL5nZmfW9Ze7+yPunufueX369GlWB9qjhSu38K/Pf8hZx/fhwatOJT1V10uISHzF8lOoCIi8rXoQUFxfm+AwVhawC7gKeNXdK9z9C+BtIA/A3YuDP78AniccOknhlQ9D/Oi5Dxg7pDd/vHYUGWmp8S5JRCSmQbIcGGpmQ8ysEzAZWFSrzSJgavB6AvC6uzvhw1nftLCuwFjgIzPrambdAYLl5wFrYtiHduP1j7bx/adXkju4J49OzSMzXSEiIu1DzKaRd/dKM7sVWAykArPdfa2ZzQIK3H0R8Bgw18w2Eh6JTA42fwiYQzgkDJjj7quDw1vPB1cnpQFPufursepDe/HWhh1898n3GT6gB3OuP42uGZr9X0TaDwsPADq2vLw8LyhIzFtOlm3aydQ575HTuyvzZ4ylZ5dO8S5JRJKEma2I5jYLnaltx1Z+vpsbHl/OwJ6deXL6GIWIiLRLCpJ2as2WUqbOfo/s7hk8ddNYsrtlxLskEZE6KUjaoY+37uXax5bRPTOdedPHcHSPzHiXJCJSLwVJO7Np+z6ufnQZ6akpzJs+hkG9usS7JBGRBilI2pHNuw5w9aPLcHeeumkMOdld412SiEijdB1pO7GnvIKrHn2XA4eqmD9jLMf17R7vkkREoqIgaSdeWLmFzbvKmD9jLMP694h3OSIiUdOhrXbimYLNDOvfgzFDjop3KSIiTaIgaQfWFpeyZsseJuUN0jNFRCThKEjagecKiuiUmkJ+bu1Z9kVE2j8FSZyVV1Tx/MotnHfi0fTqqjvXRSTxKEji7G/rtlFaVsGk0wY33lhEpB1SkMTZswWbGdizM+OOzY53KSIizaIgiaOi3Qd4a+MOJowapGeui0jCUpDE0YIVRQBcmTcozpWIiDSfgiROqqud5wqKGHdstubTEpGEpiCJk7c/2cGWkjIm6iS7iCQ4BUmcPFtQRFbndM4bfnS8SxERaREFSRyUHDjE4rVbuSx3AJnpqfEuR0SkRRQkcfDCqmIOVVbrsJaIdAgKkjh4ZvlmThrYgxMHZMW7FBGRFlOQtLE1W0pZF9rDxDyNRkSkY1CQtLFnCzbTKS2F/FM0QaOIdAwKkjZUXlHFwpVbuPCkfmR1SY93OSIirUJB0oYWr93KnvJKHdYSkQ5FQdKGni3YzOCjOvONY3rHuxQRkVajIGkjm3cd4O2NO7ly1GBN0CgiHYqCpI08V7AZM/j2KE3QKCIdi4KkDVRVOwtWFHHG0D4M7Nk53uWIiLQqBUkbeGvjDopLy5mkk+wi0gEpSNrAs8s306tLOt8a3jfepYiItDoFSYzt2n+I19Zt5bJTB5KRpgkaRaTjUZDE2MKVW6iociZpgkYR6aAUJDHk7jxbsJkRg7I4oV+PeJcjIhITCpIY+nBLKR9t3as72UWkQ1OQxNAzyzeTkZbCpbkD4l2KiEjMxDRIzOwCM/vYzDaa2cw61meY2TPB+mVmlhMsTzezJ8zsQzNbb2b/Gu0+24uyQ1UsWlXMRSf3p0emJmgUkY4rZkFiZqnAQ8CFwHBgipkNr9XsRmC3ux8H3AfcHSy/Eshw95OBUcB3zCwnyn22C6+uDbH3oCZoFJGOL5YjktHARnff5O6HgPlAfq02+cATwesFwHgzM8CBrmaWBnQGDgF7otxnu/DM8s18rXcXxh5zVLxLERGJqaiDxMwyzexGM/u+mUUzfe1AYHPE+6JgWZ1t3L0SKAV6Ew6V/UAI+By4x913RbnPmnpnmFmBmRVs3749inJbz2c79/Pupl1cOWoQ4VwUEem4mjIi+Q8gDSgHFkbRvq5PUI+yzWigChgADAF+ZGbHRLnP8EL3R9w9z93z+vTpE0W5ree5giJSDCaM0mEtEen46g0SM3vKzI6NWHQUMA94GugVxb6LgMhP0kFAcX1tgsNYWcAu4CrgVXevcPcvgLeBvCj3GVc1EzSedXwf+mVlxrscEZGYa2hE8hPgl2Z2j5llAfcAi4DXgF9Ese/lwFAzG2JmnYDJwfaRFgFTg9cTgNfd3QkfzvqmhXUFxgIfRbnPuHpjw3a27inXSXYRSRpp9a1w903AVWZ2OvAM8BJwrrtXRbNjd680s1uBxUAqMNvd15rZLKDA3RcBjwFzzWwj4ZHI5GDzh4A5wBrCh7PmuPtqgLr22dROx9KzyzfTu2snxg87Ot6liIi0iXqDxMx6ET7EVAFMBC4DFpvZ/e7+YjQ7d/eXgZdrLftZxOtywpf61t5uX13L69tne7Fz30H+vn4bU7+RQ6c03espIsmhoU+7hcBBIBOY6+5/Bi4BRplZuzqc1F4s+qCYiipnoiZoFJEkUu+IhPBluE8Rvo/jOgB3LwPuNLP+bVBbwlkf2kOf7hkcf3T3eJciItJmGgqSnwF/I3wZ7hFTkbh7KJZFJapQaTkDdKWWiCSZhk62/xX4axvWkvCKS8oY2lejERFJLjoj3ErcnVBpOf17akQiIslFQdJK9pRVcuBQFQOyOse7FBGRNtVgkJhZqpnd3lbFJLLQnjIAjUhEJOk0GCTBzYftcnbd9iZUUg5Af41IRCTJNHTVVo23zexBwne3769Z6O7vx6yqBFRcGh6RDNCIRESSTDRB8s/Bn7MiljnwzdYvJ3GFSspJTTH6dleQiEhyaTRI3P2ctigk0RWXltG3ewapKXr+iIgkl0av2jKzLDO7t+YhUWb2u2A2YIkQKimnv25GFJEkFM3lv7OBvYQnbpxI+JG3c2JZVCIKlZbRv6dOtItI8onmHMmx7v7tiPd3mtmqWBWUiGpuRjx3uKaOF5HkE82IpCx4JgkAZjYOKItdSYln94EKDlZW69JfEUlK0YxIvgv8OeK8yG6+fKqhEJ5jC3Tpr4gkpwaDxMxSgK+7+ylm1gPA3fe0SWUJJFSqmxFFJHk1dmd7NXBr8HqPQqRuoVJNjyIiySuacyR/M7P/ZWaDzeyomq+YV5ZAikvKSU81srtmxLsUEZE2F805khuCP78XscyBY1q/nMQUKi3j6B6ZpOhmRBFJQtGcI7nG3d9uo3oSUqikXNPHi0jSiuYcyT1tVEvCCu0p0/kREUla0Zwjec3Mvm1mOm5Th+pqZ2tpua7YEpGkFc05kh8CXYEqMysDDHB37xHTyhLEjv0Hqahy3UMiIkkrmtl/u7dFIYlKD7QSkWQXzey/ZmbXmNlPg/eDzWx07EtLDIfvIdHMvyKSpKI5R/L/gG8AVwXv9wEPxayiBFN8eESiIBGR5BTNOZIx7j7SzFYCuPtuM+sU47oSRqi0jIy0FI7qqm+JiCSnaEYkFWaWSvgmRMysD1Ad06oSSHFp+IFWuqhNRJJVNEHyAPA80NfM7gLeAn4d06oSiC79FZFkF81VW/PMbAUwnvClv5e5+/qYV5YgQiVljD22d7zLEBGJm2jOkeDuHwEfxbiWhFNV7Wzbe1DTo4hIUovm0JbU44u95VRVu6ZHEZGkpiBpgZpLfzUiEZFkpiBpgZqbEfvpHhIRSWIxDRIzu8DMPjazjWY2s471GWb2TLB+mZnlBMuvNrNVEV/VZpYbrFsa7LNmXd9Y9qEhIY1IRERiFyTBvScPARcCw4EpZja8VrMbgd3ufhxwH3A3hK8Uc/dcd88FrgUK3X1VxHZX16x39y9i1YfGhErL6dIplR6do7pmQUSkQ4rliGQ0sNHdN7n7IWA+kF+rTT7wRPB6ATC+junqpwBPx7DOZguVlulmRBFJerEMkoHA5oj3RcGyOtu4eyVQCtS+KWMSXw2SOcFhrZ/W95wUM5thZgVmVrB9+/bm9qFBxaXlDOipw1oiktxiGSR1fcB7U9qY2RjggLuviVh/tbufDJwRfF1b11/u7o+4e5675/Xp06dplUcpVFKmyRpFJOnFMkiKgMER7wcBxfW1MbM0IAvYFbF+MrVGI+6+JfhzL/AU4UNobe5QZTXb9x3U9CgikvRiGSTLgaFmNiSYLXgysKhWm0XA1OD1BOB1d6+ZHDIFuJLwuRWCZWlmlh28TgcuBtYQB9v2lOOOnowoIkkvZpcbuXulmd0KLAZSgdnuvtbMZgEF7r4IeAyYa2YbCY9EJkfs4kygyN03RSzLABYHIZIK/B34U6z60JBQafjS334akYhIkovpdavu/jLwcq1lP4t4XU541FHXtkuBsbWW7QdGtXqhzVBzM+IAnSMRkSSnO9ubqWZE0l9XbYlIklOQNFOopIzumWl0y9DNiCKS3BQkzVRcWq6pUUREUJA0W6i0TNPHi4igIGm2UIkesSsiAgqSZimvqGLn/kO6YktEBAVJs2w9fA+JgkREREHSDDWX/mrCRhERBUmz1NyMqAkbRUQUJM1y+GZEnWwXEVGQNEdxSRm9uqTTuVNqvEsREYk7BUkzhEp16a+ISA0FSTMUl5Rp+ngRkYCCpBk0IhER+ZKCpIkOHKqktKxC95CIiAQUJE305T0kChIREVCQNFmoRJf+iohEUpA0UfHhJyMqSEREQEHSZDUjkqOzMuJciYhI+6AgaaJQaRnZ3TLISNPNiCIioCBpsuLScp1oFxGJoCBpolBJmSZrFBGJoCBpIt2MKCJyJAVJE+wtr2DfwUqNSEREIihImuDw9PF6oJWIyGEKkiYoLqm5h0QjEhGRGgqSJtCIRETkqxQkTRAqKSPF4OjuuhlRRKSGgqQJikvL6ds9k7RUfdtERGroE7EJQqVl9NfNiCIiR1CQNEH4HhIFiYhIJAVJlNydUIluRhQRqU1BEqXSsgrKKqo0IhERqUVBEqXikponI2pEIiISSUESpVDwQCuNSEREjhTTIDGzC8zsYzPbaGYz61ifYWbPBOuXmVlOsPxqM1sV8VVtZrnBulFm9mGwzQNmZrHsQ43iUo1IRETqErMgMbNU4CHgQmA4MMXMhtdqdiOw292PA+4D7gZw93nunuvuucC1QKG7rwq2eRiYAQwNvi6IVR8ihUrKSEsxsrvpZkQRkUixHJGMBja6+yZ3PwTMB/JrtckHngheLwDG1zHCmAI8DWBm/YEe7v4/7u7An4HLYtWBSKHSco7ukUlqSpsMgEREEkYsg2QgsDnifVGwrM427l4JlAK9a7WZRBAkQfuiRvYJgJnNMLMCMyvYvn17szoQKVSqB1qJiNQllkFS13/dvSltzGwMcMDd1zRhn+GF7o+4e5675/Xp0yeaehsUKi3XZI0iInWIZZAUAYMj3g8CiutrY2ZpQBawK2L9ZL4cjdS0H9TIPluduxMqLdf08SIidYhlkCwHhprZEDPrRDgUFtVqswiYGryeALwenPvAzFKAKwmfWwHA3UPAXjMbG5xLuQ54IYZ9AGDn/kMcqqzWoS0RkTqkxWrH7l5pZrcCi4FUYLa7rzWzWUCBuy8CHgPmmtlGwiORyRG7OBMocvdNtXZ9M/A40Bl4JfiKqVCJnkMiIlKfmAUJgLu/DLxca9nPIl6XEx511LXtUmBsHcsLgJNatdBGFJfWPBlRQSIiUpvubI9CKHjErqaQFxH5KgVJFEJ7yumUmkLvrp3iXYqISLujIIlCqKScflmZtNFsLCIiCUVBEgXdjCgiUj8FSRSKS8o1WaOISD0UJI2oqna27dEjdkVE6qMgacSOfQeprHbdQyIiUg8FSSOKS2ruIdGIRESkLgqSRoSCB1r1182IIiJ1iumd7R1B6PCTETUiEWmuiooKioqKKC8vj3cpUofMzEwGDRpEenp6s7ZXkDQiVFJGZnoKWZ2b9w0WESgqKqJ79+7k5OTofqx2xt3ZuXMnRUVFDBkypFn70KGtRoSnj++sX36RFigvL6d37976d9QOmRm9e/du0WhRQdKI4tIyzbEl0goUIu1XS382CpJGhErKdaJdRKQBCpIGVFZV88VePRlRJNGdffbZLF68+Ihl999/P7fcckuD23Xr1g2A4uJiJkyYUO++CwoKGtzP/fffz4EDBw6/v+iiiygpKYmm9FZTWFjIU089FZN9K0gasG3vQapdD7QSSXRTpkxh/vz5RyybP38+U6ZMiWr7AQMGsGDBgmb//bWD5OWXX6Znz57N3l9zxDJIdNVWA7YGD7TS9CgirefO/1zLuuI9rbrP4QN68PNLTqx3/YQJE/jJT37CwYMHycjIoLCwkOLiYk4//XT27dtHfn4+u3fvpqKigl/96lfk5+cfsX1hYSEXX3wxa9asoaysjOuvv55169YxbNgwysrKDre7+eabWb58OWVlZUyYMIE777yTBx54gOLiYs455xyys7NZsmQJOTk5FBQUkJ2dzb333svs2bMBmD59Oj/4wQ8oLCzkwgsv5PTTT+edd95h4MCBvPDCC3TufOR/ap977jnuvPNOUlNTycrK4o033qCqqoqZM2eydOlSDh48yPe+9z2+853vMHPmTNavX09ubi5Tp07l9ttvb7Xvv4KkAcUlNfeQaEQiksh69+7N6NGjefXVV8nPz2f+/PlMmjQJMyMzM5Pnn3+eHj16sGPHDsaOHcull15a7wnohx9+mC5durB69WpWr17NyJEjD6+76667OOqoo6iqqmL8+PGsXr2a2267jXvvvZclS5aQnZ19xL5WrFjBnDlzWLZsGe7OmDFjOOuss+jVqxcbNmzg6aef5k9/+hMTJ07kL3/5C9dcc80R28+aNYvFixczcODAw4fKHnvsMbKysli+fDkHDx5k3LhxnHfeefzmN7/hnnvu4cUXX2zl766CpEGhYETSTyMSkVbT0MghlmoOb9UESc0owN254447eOONN0hJSWHLli1s27aNfv361bmfN954g9tuuw2AESNGMGLEiMPrnn32WR555BEqKysJhUKsW7fuiPW1vfXWW1x++eV07doVgCuuuII333yTSy+9lCFDhpCbmwvAqFGjKCws/Mr248aNY9q0aUycOJErrrgCgNdee43Vq1cfPhRXWlrKhg0b6NQpdg/mU5A0oLiknG4ZafTI1M2IIonusssu44c//CHvv/8+ZWVlh0cS8+bNY/v27axYsYL09HRycnIavaeirtHKp59+yj333MPy5cvp1asX06ZNa3Q/7l7vuoyMjMOvU1NTjziEVuMPf/gDy5Yt46WXXiI3N5dVq1bh7vz+97/n/PPPP6Lt0qVLG6ylJXSyvQF6oJVIx9GtWzfOPvtsbrjhhiNOspeWltK3b1/S09NZsmQJn332WYP7OfPMM5k3bx4Aa9asYfXq1QDs2bOHrl27kpWVxbZt23jllVcOb9O9e3f27t1b574WLlzIgQMH2L9/P88//zxnnHFG1H365JNPGDNmDLNmzSI7O5vNmzdz/vnn8/DDD1NRUQHAP/7xD/bv319vDa1BI5IGhErLdcWWSAcyZcoUrrjiiiOu4Lr66qu55JJLyMvLIzc3lxNOOKHBfdx8881cf/31jBgxgtzcXEaPHg3AKaecwqmnnsqJJ57IMcccw7hx4w5vM2PGDC688EL69+/PkiVLDi8fOXIk06ZNO7yP6dOnc+qpp9Z5GKsuP/7xj9mwYQPuzvjx4znllFMYMWIEhYWFjBw5EnenT58+LFy4kBEjRpCWlsYpp5zCtGnTWvVkuzU0tOoo8vLyvLHrvOvyyxfX0T8rk+lXgLMXAAAGbklEQVRnHBODqkSSx/r16xk2bFi8y5AG1PUzMrMV7p7X2LYakTTgpxcPj3cJIiLtns6RiIhIiyhIRKRNJMNh9ETV0p+NgkREYi4zM5OdO3cqTNqhmueRZGY2/wpVnSMRkZgbNGgQRUVFbN++Pd6lSB1qnpDYXAoSEYm59PT0Zj99T9o/HdoSEZEWUZCIiEiLKEhERKRFkuLOdjPbDjQ8gc6RsoEdMSqnvUrGPkNy9jsZ+wzJ2e+W9vlr7t6nsUZJESRNZWYF0UwL0JEkY58hOfudjH2G5Ox3W/VZh7ZERKRFFCQiItIiCpK6PRLvAuIgGfsMydnvZOwzJGe/26TPOkciIiItohGJiIi0iIJERERaJKmDxMwuMLOPzWyjmc2sY32GmT0TrF9mZjltX2XriqLPPzSzdWa22sz+y8y+Fo86W1tj/Y5oN8HM3MwS/jLRaPpsZhODn/daM3uqrWuMhSh+x//JzJaY2crg9/yieNTZWsxstpl9YWZr6llvZvZA8P1YbWYjW70Id0/KLyAV+AQ4BugEfAAMr9XmFuAPwevJwDPxrrsN+nwO0CV4fXOi9znafgftugNvAO8CefGuuw1+1kOBlUCv4H3feNfdRv1+BLg5eD0cKIx33S3s85nASGBNPesvAl4BDBgLLGvtGpJ5RDIa2Ojum9z9EDAfyK/VJh94Ini9ABhvZtaGNba2Rvvs7kvc/UDw9l2g+XNLtx/R/KwBfgn8Fihvy+JiJJo+3wQ85O67Adz9izauMRai6bcDPYLXWUBxG9bX6tz9DWBXA03ygT972LtATzPr35o1JHOQDAQ2R7wvCpbV2cbdK4FSoHebVBcb0fQ50o2E/yeT6Brtt5mdCgx29xfbsrAYiuZnfTxwvJm9bWbvmtkFbVZd7ETT718A15hZEfAy8P22KS1umvrvvsmS+XkkdY0sal8LHU2bRBJ1f8zsGiAPOCumFbWNBvttZinAfcC0tiqoDUTzs04jfHjrbMIjzzfN7CR3L4lxbbEUTb+nAI+7++/M7BvA3KDf1bEvLy5i/jmWzCOSImBwxPtBfHWIe7iNmaURHgY3NIRs76LpM2b2LeDfgEvd/WAb1RZLjfW7O3ASsNTMCgkfR16U4Cfco/39fsHdK9z9U+BjwsGSyKLp943AswDu/j9AJuHJDTuqqP7dt0QyB8lyYKiZDTGzToRPpi+q1WYRMDV4PQF43YOzVwmq0T4Hh3j+SDhEOsIxc2ik3+5e6u7Z7p7j7jmEzw1d6u4F8Sm3VUTz+72Q8MUVmFk24UNdm9q0ytYXTb8/B8YDmNkwwkHSkZ8BvAi4Lrh6ayxQ6u6h1vwLkvbQlrtXmtmtwGLCV3rMdve1ZjYLKHD3RcBjhIe9GwmPRCbHr+KWi7LP/w50A54Lriv43N0vjVvRrSDKfncoUfZ5MXCema0DqoAfu/vO+FXdclH2+0fAn8zsdsKHeKYl8n8Qzexpwocns4PzPj8H0gHc/Q+EzwNdBGwEDgDXt3oNCfz9ExGRdiCZD22JiEgrUJCIiEiLKEhERKRFFCQiItIiChIREWkRBYlIM5hZPzObb2afBLPnvmxmx7fCfve1Rn0ibUlBItJEwcSdzwNL3f1Ydx8O3AEcHd/KROJDQSLSdOcAFcHNXgC4+yp3fzOykZndbWa3RLz/hZn9yMy6Bc96ed/MPjSzr8xEbGZnm9mLEe8fNLNpwetRZvbfZrbCzBa39kyuIk2lIBFpupOAFVG0mw9Ming/EXiO8DT1l7v7SMKh9LtoH09gZunA74EJ7j4KmA3c1YTaRVpd0k6RIhJr7r7SzPqa2QCgD7Db3T8PwuDXZnYmUE14Su+jga1R7PbrhIPsb0H2pAKtOm+SSFMpSESabi3hSTyjsSBo24/wCAXgasLBMsrdK4IZhzNrbVfJkUcMatYbsNbdv9GMukViQoe2RJrudSDDzG6qWWBmp5lZXc9umU94ss8JhEMFwo8j+CIIkXOAr9Wx3WfAcDPLMLMsgtlqCU/13id4jgZmlm5mJ7ZKr0SaSUEi0kTBTLGXA+cGl/+uJfzUva8848Hd1xJ+3smWiKm75wF5ZlZAeHTyUR3bbSb8zIzVQfuVwfJDhEPpbjP7AFgF/HOrdlCkiTT7r4iItIhGJCIi0iIKEhERaREFiYiItIiCREREWkRBIiIiLaIgERGRFlGQiIhIi/x/GHtHjgONUBUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp = dict(zip(C, mnist_val_err))\n",
    "msg = \"\"\n",
    "for c, err in temp.items():\n",
    "    msg += str(c) + \": \" + str(err) + \"\\n\"\n",
    "print(\"C values tried with corresponding validation errors\"+ \n",
    "      \"(C = key, error = value): \\n\" + msg)\n",
    "print(\"Best C value for MNIST is: \" + \n",
    "      str(C[mnist_val_err.index(min(mnist_val_err))]))\n",
    "plt.plot(C, mnist_val_err,label=\"Validation set\")\n",
    "plt.xlabel(\"C value\")\n",
    "plt.ylabel(\"error %\")\n",
    "plt.title(\"MNIST\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\pagebreak "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #5 K-Fold Cross-Validation\n",
    "#### Here I do 5-fold cross-validation on the entire training set portion of the SPAM dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get entire training set back\n",
    "spam = io.loadmat(\"data/%s_data.mat\" % \"spam\")\n",
    "k = 5\n",
    "spam_x, spam_y = spam[\"training_data\"], spam[\"training_labels\"]\n",
    "fold_size = spam_x.shape[0]/k\n",
    "\n",
    "def spam_folds_train(c):\n",
    "    accs = []\n",
    "    for i in range(k):\n",
    "        val_begin = int(fold_size*i)\n",
    "        val_end = int(fold_size*(i+1))\n",
    "        spam_xval, spam_yval = \\\n",
    "        spam_x[val_begin:val_end], spam_y[val_begin:val_end]\n",
    "        spam_xt, spam_yt = \\\n",
    "        np.vstack((spam_x[:val_begin],spam_x[val_end:])),\\\n",
    "        np.vstack((spam_y[:val_begin],spam_y[val_end:]))\n",
    "        accs.append(train_hyper(spam_xt, spam_yt, spam_xval, \n",
    "                                spam_yval, spam_xt.shape[0], \"linear\", c))\n",
    "    return accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_folds_val_acc = [spam_folds_train(c) for c in C]\n",
    "spam_folds_val_acc = [sum(i)/len(i) for i in spam_folds_val_acc]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The cross-validation scores for various and best C value(s) is/are listed/printed below the following code cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C values tried with corresponding Cross-Validation accuracies\n",
      "(C = key, accuracy = value): \n",
      "0.01: 0.2839301432455919\n",
      "0.05: 0.25182033096926715\n",
      "0.1: 0.24814603014417996\n",
      "0.25: 0.2433098795540979\n",
      "0.5: 0.23944140760051952\n",
      "0.75: 0.2388602023939675\n",
      "1: 0.23905325222624024\n",
      "\n",
      "Best C value: 0.75\n"
     ]
    }
   ],
   "source": [
    "temp = dict(zip(C, spam_folds_val_acc))\n",
    "msg = \"\"\n",
    "for c, acc in temp.items():\n",
    "    msg += str(c) + \": \" + str(acc) + \"\\n\"\n",
    "print(\"C values tried with corresponding Cross-Validation accuracies\"+\n",
    "      \"\\n(C = key, accuracy = value): \\n\" + msg)\n",
    "print(\"Best C value: \" + str(C[spam_folds_val_acc.index(min(spam_folds_val_acc))]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\pagebreak "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #6 Kaggle\n",
    "#### Here I prepare to submit prediction to Kaggle through multiple steps.\n",
    "#### 1) I preprocess if necessary (i.e. MNIST rescale/standardize). \n",
    "#### 2) I find the best set of hyperparameters through trying out combos. \n",
    "#### 3) I use my findings from steps 1 and 2 to generate predictions. Note that I modified the provided \"results_to_csv\" function so I could give clearer names to the generated files. \n",
    "#### Kaggle username: danish123\n",
    "#### Kaggle MNIST score: 0.97000\n",
    "#### Kaggle SPAM score: 0.80535\n",
    "#### Kaggle CIFAR-10 score: 0.40500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_kaggle_vals = [] #validation values\n",
    "C = [0.01, 0.5, 1.0]\n",
    "gammas = [0.001, 0.01, 0.1]\n",
    "kernels = [\"poly\",\"rbf\"]\n",
    "for c in C:\n",
    "    for gamma in gammas:\n",
    "        for kernel in kernels:\n",
    "            val_err = \\\n",
    "            train_hyper(mnist_xtrain, mnist_ytrain, mnist_xval, mnist_yval, \\\n",
    "                        2000, kernel, c, gamma)\n",
    "            mnist_kaggle_vals.append((val_err,kernel,c,gamma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best combo (kernel,c,gamma)(0.0625, 'poly', 0.01, 0.1)\n"
     ]
    }
   ],
   "source": [
    "print(\"best combo (kernel,c,gamma)\"+str(min(mnist_kaggle_vals, key = lambda x: x[0]))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(data_xtrain, data_ytrain, data_xtest, num_ex, kernel, C, gamma):\n",
    "    data_ytrain = data_ytrain.reshape(-1,)\n",
    "    model = svm.SVC(cache_size=4000,kernel=kernel,C=C,gamma=gamma)\n",
    "    model.fit(data_xtrain[:num_ex,:],data_ytrain[:num_ex])\n",
    "    return model.predict(data_xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_xtest = io.loadmat(\"data/%s_data.mat\" % \"mnist\")[\"test_data\"]\n",
    "mnist_xtest = (mnist_xtest-np.mean(mnist_xtest))/np.std(mnist_xtest)\n",
    "results_to_csv(test(mnist_xtrain, mnist_ytrain, mnist_xtest, \\\n",
    "                    10000, \"poly\", 0.01, 0.1),\"mnist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_kaggle_vals = [] #validation values\n",
    "C = [0.01, 0.5, 1, 5, 10, 100, 500, 1000]\n",
    "gammas = [0.001, 0.01, 0.1]\n",
    "kernels = [\"poly\",\"rbf\"]\n",
    "for c in C:\n",
    "    for gamma in gammas:\n",
    "        for kernel in kernels:\n",
    "            val_err = \\\n",
    "            train_hyper(spam_xtrain, spam_ytrain, spam_xval, spam_yval, \\\n",
    "                        2000, kernel, c, gamma)\n",
    "            spam_kaggle_vals.append((val_err,kernel,c,gamma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best combo (kernel,c,gamma)(0.17601547388781436, 'rbf', 1000, 0.001)\n"
     ]
    }
   ],
   "source": [
    "print(\"best combo (kernel,c,gamma)\"+str(min(spam_kaggle_vals, key = lambda x: x[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_xtest = io.loadmat(\"data/%s_data.mat\" % \"spam\")[\"test_data\"]\n",
    "results_to_csv(test(spam_xtrain, spam_ytrain, spam_xtest,\\\n",
    "                    10000, \"rbf\", 1000, 0.001),\"spam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this takes much longer to train than the above 2 datasets...\n",
    "cifar10_kaggle_vals = [] #validation values\n",
    "C = [0.01, 0.5, 1.0]\n",
    "gammas = [0.001, 0.01, 0.1]\n",
    "kernels = [\"poly\",\"rbf\"]\n",
    "for c in C:\n",
    "    for gamma in gammas:\n",
    "        for kernel in kernels:\n",
    "            val_err = \\\n",
    "            train_hyper(cifar10_xtrain, cifar10_ytrain, cifar10_xval, cifar10_yval, \\\n",
    "                        2000, kernel, c, gamma)\n",
    "            cifar10_kaggle_vals.append((val_err,kernel,c,gamma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best combo (kernel,c,gamma)(0.6635, 'poly', 0.01, 0.001)\n"
     ]
    }
   ],
   "source": [
    "min_index = cifar10_kaggle_vals.index(min(cifar10_kaggle_vals))\n",
    "print(\"best combo (kernel,c,gamma)\"\n",
    "      +str(min(cifar10_kaggle_vals, key = lambda x: x[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_xtest = io.loadmat(\"data/%s_data.mat\" % \"cifar10\")[\"test_data\"]\n",
    "results_to_csv(test(cifar10_xtrain, cifar10_ytrain, cifar10_xtest, 10000, \n",
    "                    \"poly\", 0.01, 0.001), \"cifar10\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
